Merge remote-tracking branch 'origin/staging' into develop

🐛 Fix problem with thumbnail regeneration

.circleci/config.yml

@@ -1,351 +0,0 @@
-version: 2.1
-
-jobs:
-  lint:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    steps:
-      - checkout
-
-      - run:
-          name: "fmt check"
-          working_directory: "."
-          command: |
-            yarn install
-            yarn run fmt:clj:check
-
-      - run:
-          name: "lint clj common"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:common
-
-      - run:
-          name: "lint clj frontend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:frontend
-
-      - run:
-          name: "lint clj backend"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:backend
-
-      - run:
-          name: "lint clj exporter"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:exporter
-
-      - run:
-          name: "lint clj library"
-          working_directory: "."
-          command: |
-            yarn run lint:clj:library
-
-  test-common:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
-
-      - run:
-          name: "JVM tests"
-          working_directory: "./common"
-          command: |
-            clojure -M:dev:test
-
-      - run:
-          name: "NODE tests"
-          working_directory: "./common"
-          command: |
-            yarn install
-            yarn run test
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "common/deps.edn"}}-{{ checksum "common/yarn.lock" }}
-
-  test-frontend:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: "install dependencies"
-          working_directory: "./frontend"
-          # We install playwright here because the dependent tasks
-          # uses the same cache as this task so we prepopulate it
-          command: |
-            yarn install
-            yarn run playwright install chromium
-
-      - run:
-          name: "lint scss on frontend"
-          working_directory: "./frontend"
-          command: |
-            yarn run lint:scss
-
-      - run:
-          name: "unit tests"
-          working_directory: "./frontend"
-          command: |
-            yarn run test
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.yarn
-            - ~/.gitlibs
-            - ~/.cache/ms-playwright
-          key: v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-  test-library:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx6g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: Install dependencies and build
-          working_directory: "./library"
-          command: |
-            yarn install
-
-      - run:
-          name: Build and Test
-          working_directory: "./library"
-          command: |
-            ./scripts/build
-            yarn run test
-
-  test-components:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx6g -Xms2g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      - run:
-          name: Install dependencies
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn run playwright install chromium
-
-      - run:
-          name: Build Storybook
-          working_directory: "./frontend"
-          command: yarn run build:storybook
-
-      - run:
-          name: Serve Storybook and run tests
-          working_directory: "./frontend"
-          command: |
-            npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
-              "npx http-server storybook-static --port 6006 --silent" \
-              "npx wait-on tcp:6006 && yarn test:storybook"
-
-  test-integration:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: large
-
-    environment:
-      JAVA_OPTS: -Xmx6g -Xms2g
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
-
-      # Build frontend
-      - run:
-          name: "frontend build"
-          working_directory: "./frontend"
-          command: |
-            yarn install
-            yarn run build:app:assets
-            yarn run build:app
-            yarn run build:app:libs
-
-          # Build the wasm bundle
-      - run:
-          name: "wasm build"
-          working_directory: "./render-wasm"
-          command: |
-            EMSDK_QUIET=1 . /opt/emsdk/emsdk_env.sh
-            ./build release
-
-      # Run integration tests
-      - run:
-          name: "integration tests"
-          working_directory: "./frontend"
-          command: |
-            yarn run playwright install chromium
-            yarn run test:e2e -x --workers=4
-
-  test-backend:
-    docker:
-      - image: penpotapp/devenv:latest
-      - image: cimg/postgres:14.5
-        environment:
-          POSTGRES_USER: penpot_test
-          POSTGRES_PASSWORD: penpot_test
-          POSTGRES_DB: penpot_test
-      - image: cimg/redis:7.0.5
-
-    working_directory: ~/repo
-    resource_class: medium+
-
-    environment:
-      JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
-      NODE_OPTIONS: --max-old-space-size=4096
-
-    steps:
-      - checkout
-
-      - restore_cache:
-          keys:
-            - v1-dependencies-{{ checksum "backend/deps.edn" }}
-
-      - run:
-          name: "tests"
-          working_directory: "./backend"
-          command: |
-            clojure -M:dev:test --reporter kaocha.report/documentation
-
-          environment:
-            PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
-            PENPOT_TEST_DATABASE_USERNAME: penpot_test
-            PENPOT_TEST_DATABASE_PASSWORD: penpot_test
-            PENPOT_TEST_REDIS_URI: "redis://localhost/1"
-
-      - save_cache:
-          paths:
-            - ~/.m2
-            - ~/.gitlibs
-          key: v1-dependencies-{{ checksum "backend/deps.edn" }}
-
-  test-render-wasm:
-    docker:
-      - image: penpotapp/devenv:latest
-
-    working_directory: ~/repo
-    resource_class: medium+
-    environment:
-
-    steps:
-      - checkout
-
-      - run:
-          name: "fmt check"
-          working_directory: "./render-wasm"
-          command: |
-            cargo fmt --check
-
-      - run:
-          name: "lint"
-          working_directory: "./render-wasm"
-          command: |
-            ./lint
-
-      - run:
-          name: "cargo tests"
-          working_directory: "./render-wasm"
-          command: |
-            ./test
-
-workflows:
-  penpot:
-    jobs:
-      - test-frontend:
-          requires:
-            - lint: success
-
-      - test-library:
-          requires:
-            - lint: success
-
-      - test-components:
-          requires:
-            - lint: success
-
-      - test-backend:
-          requires:
-            - lint: success
-
-      - test-common:
-          requires:
-            - lint: success
-
-      - lint
-      - test-integration
-      - test-render-wasm

.github/workflows/build-bundle.yml

@@ -84,8 +84,10 @@ jobs:
         uses: mattermost/action-mattermost-notify@master
         with:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
           TEXT: |
-            ❌ *[PENPOT] Error during the execution of the job*
+            ❌ 📦 *[PENPOT] Error building penpot bundles.*
             📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
                Bundle version: `${{ steps.vars.outputs.bundle_version }}`
             🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/build-docker.yml

@@ -34,18 +34,26 @@ jobs:
           echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
 
       - name: Download Penpot Bundles
+        id: bundles
         env:
           FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
         run: |
+          tmp=$(aws s3api head-object \
+            --bucket ${{ secrets.S3_BUCKET }} \
+            --key "$FILE_NAME" \
+            --query 'Metadata."bundle-version"' \
+            --output text)
+          echo "bundle_version=$tmp" >> $GITHUB_OUTPUT
           pushd docker/images
           aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
           unzip $FILE_NAME > /dev/null
           mv penpot/backend bundle-backend
           mv penpot/frontend bundle-frontend
           mv penpot/exporter bundle-exporter
+          mv penpot/storybook bundle-storybook
           popd
 
       - name: Set up Docker Buildx
@@ -58,6 +66,18 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images:
+            frontend
+            backend
+            exporter
+            storybook
+          labels: |
+            bundle_version=${{ steps.bundles.outputs.bundle_version }}
+
       - name: Build and push Backend Docker image
         uses: docker/build-push-action@v6
         env:
@@ -69,6 +89,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
 
@@ -83,6 +104,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
 
@@ -97,5 +119,34 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push Storybook Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'storybook'
+          BUNDLE_PATH: './bundle-storybook'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.storybook
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🐳 *[PENPOT] Error building penpot docker images.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            📦 Bundle: `${{ steps.bundles.outputs.bundle_version }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/build-nitrate-module.yml

@@ -0,0 +1,21 @@
+name: _NITRATE MODULE
+
+on:
+  schedule:
+    - cron: '36 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "nitrate-module"
+      build_wasm: "yes"
+      build_storybook: "yes"
+
+  build-docker:
+    needs: build-bundle
+    uses: ./.github/workflows/build-docker.yml
+    secrets: inherit
+    with:
+      gh_ref: "nitrate-module"

.github/workflows/build-staging-render.yml

@@ -0,0 +1,14 @@
+name: _STAGING RENDER
+
+on:
+  schedule:
+    - cron: '36 5-20 * * 1-5'
+
+jobs:
+  build-bundle:
+    uses: ./.github/workflows/build-bundle.yml
+    secrets: inherit
+    with:
+      gh_ref: "staging-render"
+      build_wasm: "yes"
+      build_storybook: "yes"

.github/workflows/build-tag.yml

@@ -11,7 +11,7 @@ jobs:
     secrets: inherit
     with:
       gh_ref: ${{ github.ref_name }}
-      build_wasm: "no"
+      build_wasm: "yes"
       build_storybook: "yes"
 
   build-docker:
@@ -21,6 +21,22 @@ jobs:
     with:
       gh_ref: ${{ github.ref_name }}
 
+  notify:
+    name: Notifications
+    runs-on: ubuntu-24.04
+    needs: build-docker
+
+    steps:
+      - name: Notify Mattermost
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            🐳 *[PENPOT] Docker image available: ${{ github.ref_name }}*
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra
+
   publish-final-tag:
     if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta')  && contains(github.ref_name, '.') }}
     needs: build-docker

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s["A-Z].*[^.]$'
+          pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert).+[^.])$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/release.yml

@@ -37,36 +37,43 @@ jobs:
           ref: ${{ steps.vars.outputs.gh_ref }}
 
       # --- Publicly release the docker images ---
-      - name: Login to private registry
+      - name: Configure ECR credentials
-        uses: docker/login-action@v3
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          registry: ${{ secrets.DOCKER_REGISTRY }}
+          aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
-          username: ${{ secrets.DOCKER_USERNAME }}
+          aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          aws-region: ${{ secrets.AWS_REGION }}
 
-      - name: Login to DockerHub
+      - name: Install Skopeo
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.PUB_DOCKER_USERNAME }}
-          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      - name: Publish docker images to DockerHub
-        env:
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-          REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
-          HUB: ${{ secrets.PUB_DOCKER_HUB }}
         run: |
-          IMAGES=("frontend" "backend" "exporter")
+          sudo apt-get update -y
-          EXTRA_TAGS=("main" "latest")
+          sudo apt-get install -y skopeo
+
+      - name: Copy images from AWS ECR to Docker Hub
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+          PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
+          PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
+          TAG: ${{ steps.vars.outputs.gh_ref }}
+        run: |
+          aws ecr get-login-password --region $AWS_REGION | \
+            skopeo login --username AWS --password-stdin \
+            $DOCKER_REGISTRY
+
+          echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io
+
+          IMAGES=("frontend" "backend" "exporter" "storybook")
 
           for image in "${IMAGES[@]}"; do
-            docker pull "$REGISTRY/penpotapp/$image:$TAG"
+            skopeo copy --all \
-            docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$TAG"
+              docker://$DOCKER_REGISTRY/$image:$TAG \
-            docker push "penpotapp/$image:$TAG"
+              docker://docker.io/penpotapp/$image:$TAG
 
-            for tag in "${EXTRA_TAGS[@]}"; do
+            for alias in main latest; do
-              docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$tag"
+              skopeo copy --all \
-              docker push "penpotapp/$image:$tag"
+                docker://$DOCKER_REGISTRY/$image:$TAG \
+                docker://docker.io/penpotapp/$image:$alias
             done
           done
 
@@ -93,3 +100,15 @@ jobs:
           tag_name: ${{ steps.vars.outputs.gh_ref }}
           name: ${{ steps.vars.outputs.gh_ref }}
           body: ${{ steps.extract_release_notes.outputs.release_notes }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🚀 *[PENPOT] Error releasing penpot.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/tests.yml

@@ -0,0 +1,336 @@
+name: "CI"
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - develop
+      - staging
+
+concurrency:
+  group: ${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: "Linter"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check clojure code format
+        run: |
+          ./scripts/lint
+
+  test-common:
+    name: "Common Tests"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests on JVM
+        working-directory: ./common
+        run: |
+          clojure -M:dev:test
+
+      - name: Run tests on NODE
+        working-directory: ./common
+        run: |
+          ./scripts/test
+
+  test-plugins:
+    name: Plugins Runtime Linter & Tests
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node
+        id: setup-node
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install deps
+        working-directory: ./plugins
+        shell: bash
+        run: |
+          corepack enable;
+          corepack install;
+          pnpm install;
+
+      - name: Run Lint
+        working-directory: ./plugins
+        run: pnpm run lint
+
+      - name: Run Format Check
+        working-directory: ./plugins
+        run: pnpm run format:check
+
+      - name: Run Test
+        working-directory: ./plugins
+        run: pnpm run test
+
+      - name: Build runtime
+        working-directory: ./plugins
+        run: pnpm run build
+
+      - name: Build plugins
+        working-directory: ./plugins
+        run: pnpm run build:plugins
+
+      - name: Build styles
+        working-directory: ./plugins
+        run: pnpm run build:styles-example
+
+  test-frontend:
+    name: "Frontend Tests"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Unit Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test
+
+      - name: Component Tests
+        working-directory: ./frontend
+        env:
+          VITEST_BROWSER_TIMEOUT: 120000
+        run: |
+          ./scripts/test-components
+
+  test-render-wasm:
+    name: "Render WASM Tests"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Format
+        working-directory: ./render-wasm
+        run: |
+          cargo fmt --check
+
+      - name: Lint
+        working-directory: ./render-wasm
+        run: |
+          ./lint
+
+      - name: Test
+        working-directory: ./render-wasm
+        run: |
+          ./test
+
+  test-backend:
+    name: "Backend Tests"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    services:
+      postgres:
+        image: postgres:17
+        # Provide the password for postgres
+        env:
+          POSTGRES_USER: penpot_test
+          POSTGRES_PASSWORD: penpot_test
+          POSTGRES_DB: penpot_test
+
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      redis:
+        image: valkey/valkey:9
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        working-directory: ./backend
+        env:
+          PENPOT_TEST_DATABASE_URI: "postgresql://postgres/penpot_test"
+          PENPOT_TEST_DATABASE_USERNAME: penpot_test
+          PENPOT_TEST_DATABASE_PASSWORD: penpot_test
+          PENPOT_TEST_REDIS_URI: "redis://redis/1"
+
+        run: |
+          clojure -M:dev:test --reporter kaocha.report/documentation
+
+  test-library:
+    name: "Library Tests"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        working-directory: ./library
+        run: |
+          ./scripts/test
+
+  build-integration:
+    name: "Build Integration Bundle"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build Bundle
+        working-directory: ./frontend
+        run: |
+          ./scripts/build 0.0.0
+
+      - name: Store Bundle Cache
+        uses: actions/cache@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+
+  test-integration-1:
+    name: "Integration Tests 1/4"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="1/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-1
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-2:
+    name: "Integration Tests 2/4"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="2/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-2
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-3:
+    name: "Integration Tests 3/4"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="3/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-3
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3
+
+  test-integration-4:
+    name: "Integration Tests 4/4"
+    runs-on: ubuntu-24.04
+    container: penpotapp/devenv:latest
+    needs: build-integration
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Restore Cache
+        uses: actions/cache/restore@v4
+        with:
+          key: "integration-bundle-${{ github.sha }}"
+          path: frontend/resources/public
+
+      - name: Run Tests
+        working-directory: ./frontend
+        run: |
+          ./scripts/test-e2e --shard="4/4";
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-tests-result-4
+          path: frontend/test-results/
+          overwrite: true
+          retention-days: 3

.gitignore

@@ -80,3 +80,4 @@ node_modules
 /playwright/.cache/
 /render-wasm/target/
 /**/.yarn/*
+/.pnpm-store

.nvmrc

@@ -1 +1 @@
-v22.19.0
+v22.21.1

CHANGES.md

@@ -1,6 +1,6 @@
 # CHANGELOG
 
-## 2.12.0 (Unreleased)
+## 2.14.0 (Unreleased)
 
 ### :boom: Breaking changes & Deprecations
 
@@ -12,10 +12,143 @@
 
 ### :bug: Bugs fixed
 
+
+## 2.13.0 (Unreleased)
+
+### :boom: Breaking changes & Deprecations
+
+### :rocket: Epics and highlights
+
+### :heart: Community contributions (Thank you!)
+
+- Fix mask issues with component swap (by @dfelinto) [Github #7675](https://github.com/penpot/penpot/issues/7675)
+
+### :sparkles: New features & Enhancements
+
+- Add new Box Shadow Tokens [Taiga #10201](https://tree.taiga.io/project/penpot/us/10201)
+- Make i18n translation files load on-demand [Taiga #11474](https://tree.taiga.io/project/penpot/us/11474)
+- Add deleted files to dashboard [Taiga #8149](https://tree.taiga.io/project/penpot/us/8149)
+
+### :bug: Bugs fixed
+
+- Fix problem when drag+duplicate a full grid [Taiga #12565](https://tree.taiga.io/project/penpot/issue/12565)
+- Fix problem when pasting elements in reverse flex layout [Taiga #12460](https://tree.taiga.io/project/penpot/issue/12460)
+- Fix wrong board size presets in Android [Taiga #12339](https://tree.taiga.io/project/penpot/issue/12339)
+- Fix problem with grid layout components and auto sizing [Github #7797](https://github.com/penpot/penpot/issues/7797)
+
+
+## 2.12.1
+
+### :bug: Bugs fixed
+
+- Fix setting a portion of text as bold or underline messes things up [Github #7980](https://github.com/penpot/penpot/issues/7980)
+- Fix problem with style in fonts input [Taiga #12935](https://tree.taiga.io/project/penpot/issue/12935)
+- Fix problem with path editor and right click [Github #7917](https://github.com/penpot/penpot/issues/7917)
+
+
+## 2.12.0
+
+### :boom: Breaking changes & Deprecations
+
+#### Backend RPC API changes
+
+The backend RPC API URLS are changed from `/api/rpc/command/<name>` to
+`/api/main/methods/<name>`. The previous PATH is preserved for backward
+compatibility; however, if you are a user of this API, it is strongly
+recommended that you adapt your code to use the new PATH.
+
+
+#### Updated SSO Callback URL
+
+The OAuth / Single Sign-On (SSO) callback endpoint has changed to
+align with the new OpenID Connect (OIDC) implementation.
+
+Old callback URL:
+
+```
+https://<your_domain>/api/auth/oauth/<oauth_provider>/callback
+```
+
+New callback URL:
+
+```
+https://<your_domain>/api/auth/oidc/callback
+```
+
+**Action required:**
+
+If you have SSO/Social-Auth configured on your on-premise instance,
+the following actions are required before update:
+
+Update your OAuth or SSO provider configuration (e.g., Okta, Google,
+Azure AD, etc.) to use the new callback URL. Failure to update may
+result in authentication failures after upgrading.
+
+**Reason for change:**
+
+This update standardizes all authentication flows under the single URL
+and makis it more modular, enabling the ability to configure SSO auth
+provider dinamically.
+
+
+#### Changes on default docker compose
+
+We have updated the `docker/images/docker-compose.yaml` with a small
+change related to the `PENPOT_SECRET_KEY`. Since this version, this
+environment variable is also required on exporter. So if you are using
+penpot on-premise you will need to apply the same changes on your own
+`docker-compose.yaml` file.
+
+We have removed the Minio server from the `docker/images/docker-compose.yml`
+example. It's still usable as before, we just removed the example.
+
+### :rocket: Epics and highlights
+
+### :heart: Community contributions (Thank you!)
+
+- Ensure consistent snap behavior across all zoom levels [Github #7774](https://github.com/penpot/penpot/pull/7774) by [@Tokytome](https://github.com/Tokytome)
+- Fix crash in token grid view due to tooltip validation (by @dfelinto) [Github #7887](https://github.com/penpot/penpot/pull/7887)
+- Enable Hindi translations on the application
+
+### :sparkles: New features & Enhancements
+
+- Add the ability to select boards to export as PDF [Taiga #12320](https://tree.taiga.io/project/penpot/issue/12320)
+- Add toggle for switching boolean property values [Taiga #12341](https://tree.taiga.io/project/penpot/us/12341)
+- Make the file export process more reliable [Taiga #12555](https://tree.taiga.io/project/penpot/us/12555)
+- Add auth flow changes [Taiga #12333](https://tree.taiga.io/project/penpot/us/12333)
+- Add new shape validation mechanism for shapes [Github #7696](https://github.com/penpot/penpot/pull/7696)
+- Apply color tokens from sidebar [Taiga #11353](https://tree.taiga.io/project/penpot/us/11353)
+- Display tokens in the inspect tab [Taiga #9313](https://tree.taiga.io/project/penpot/us/9313)
+- Refactor clipboard behavior to assess some minor inconsistencies and make pasting binary data faster. [Taiga #12571](https://tree.taiga.io/project/penpot/task/12571)
+
+### :bug: Bugs fixed
+
+- Fix text line-height values are wrong [Taiga #12252](https://tree.taiga.io/project/penpot/issue/12252)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
 - Fix pan cursor not disabling viewport guides [Github #6985](https://github.com/penpot/penpot/issues/6985)
 - Fix viewport resize on locked shapes [Taiga #11974](https://tree.taiga.io/project/penpot/issue/11974)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix on copy instance inside a components chain touched are missing [Taiga #12371](https://tree.taiga.io/project/penpot/issue/12371)
+- Fix problem with multiple selection and shadows [Github #7437](https://github.com/penpot/penpot/issues/7437)
+- Fix search shortcut [Taiga #10265](https://tree.taiga.io/project/penpot/issue/10265)
+- Fix shortcut conflict in text editor (increase/decrease font size vs word selection)
+- Fix problem with plugins generating code for pages different than current one [Taiga #12312](https://tree.taiga.io/project/penpot/issue/12312)
+- Fix input confirmation behavior is not uniform [Taiga #12294](https://tree.taiga.io/project/penpot/issue/12294)
+- Fix copy/pasting application/transit+json [Taiga #12721](https://tree.taiga.io/project/penpot/issue/12721)
+- Fix problem with plugins content attribute [Plugins #209](https://github.com/penpot/penpot-plugins/issues/209)
+- Fix U and E icon displayed in project list [Taiga #12806](https://tree.taiga.io/project/penpot/issue/12806)
+- Fix unpublish library modal not scrolling a long file list [Taiga #12285](https://tree.taiga.io/project/penpot/issue/12285)
+- Fix incorrect interaction betwen hower and scroll on assets sidebar [Taiga #12389](https://tree.taiga.io/project/penpot/issue/12389)
+- Fix switch variants with paths [Taiga #12841](https://tree.taiga.io/project/penpot/issue/12841)
+- Fix referencing typography tokens on font-family tokens [Taiga #12492](https://tree.taiga.io/project/penpot/issue/12492)
+- Fix horizontal scroll on layer panel [Taiga #12843](https://tree.taiga.io/project/penpot/issue/12843)
+- Fix unicode handling on email template abbreviation filter [Github #7966](https://github.com/penpot/penpot/pull/7966)
 
-## 2.11.0 (Unreleased)
+## 2.11.1
+
+- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
+
+## 2.11.0
 
 ### :boom: Breaking changes & Deprecations
 
@@ -43,10 +176,6 @@
   services which use netty internally (redis connection, S3 SDK client). This
   configuration is not very commonly used so don't expected real impact on any user.
 
-### :rocket: Epics and highlights
-
-### :heart: Community contributions (Thank you!)
-
 ### :sparkles: New features & Enhancements
 
 - New composite token: Typography [Taiga #10200](https://tree.taiga.io/project/penpot/us/10200)
@@ -56,6 +185,7 @@
 - Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
 - Reorder properties for a component [Taiga #10225](https://tree.taiga.io/project/penpot/us/10225)
 - File Data storage layout refactor [Github #7345](https://github.com/penpot/penpot/pull/7345)
+- Make several queries optimization on comment threads [Github #7506](https://github.com/penpot/penpot/pull/7506)
 
 ### :bug: Bugs fixed
 
@@ -70,7 +200,29 @@
 - Fix auto-width changes to fixed when switching variants [Taiga #12172](https://tree.taiga.io/project/penpot/issue/12172)
 - Fix component number has no singular translation string [Taiga #12106](https://tree.taiga.io/project/penpot/issue/12106)
 - Fix adding/removing identical text fills [Taiga #12287](https://tree.taiga.io/project/penpot/issue/12287)
-
+- Fix scroll on the inspect tab [Taiga #12293](https://tree.taiga.io/project/penpot/issue/12293)
+- Fix lock proportion tooltip [Taiga #12326](https://tree.taiga.io/project/penpot/issue/12326)
+- Fix internal Error when selecting a set by name in the token theme editor [Taiga #12310](https://tree.taiga.io/project/penpot/issue/12310)
+- Fix drag & drop functionality is swapping instead or reordering [Taiga #12254](https://tree.taiga.io/project/penpot/issue/12254)
+- Fix variants not syncronizing tokens on switch [Taiga #12290](https://tree.taiga.io/project/penpot/issue/12290)
+- Fix incorrect behavior of Alt + Drag for variants [Taiga #12309](https://tree.taiga.io/project/penpot/issue/12309)
+- Fix text override is lost after switch [Taiga #12269](https://tree.taiga.io/project/penpot/issue/12269)
+- Fix exporting a board crashing the app [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12384)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix selected colors not showing colors from children shapes in multiple selection [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12385)
+- Fix scrollbar issue in design tab [Taiga #12367](https://tree.taiga.io/project/penpot/issue/12367)
+- Fix library update notificacions showing when they should not [Taiga #12397](https://tree.taiga.io/project/penpot/issue/12397)
+- Fix remove flex button doesn’t work within variant [Taiga #12314](https://tree.taiga.io/project/penpot/issue/12314)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
+- Fix problem with certain text input in some editable labels (pages, components, tokens...) being in conflict with the drag/drop functionality [Taiga #12316](https://tree.taiga.io/project/penpot/issue/12316)
+- Fix not controlled theme renaming [Taiga #12411](https://tree.taiga.io/project/penpot/issue/12411)
+- Fix paste without selection sends the new element in the back [Taiga #12382](https://tree.taiga.io/project/penpot/issue/12382)
+- Fix options button does not work for comments created in the lower part of the screen [Taiga #12422](https://tree.taiga.io/project/penpot/issue/12422)
+- Fix problem when checking usage with removed teams [Taiga #12442](https://tree.taiga.io/project/penpot/issue/12442)
+- Fix focus mode persisting across page/file navigation [Taiga #12469](https://tree.taiga.io/project/penpot/issue/12469)
+- Fix shadow color validation [Github #7705](https://github.com/penpot/penpot/pull/7705)
+- Fix exception on selection blend-mode using keyboard [Github #7710](https://github.com/penpot/penpot/pull/7710)
+- Fix crash when using decimal (floating-point) values for X/Y or width/height [Taiga #12543](https://tree.taiga.io/project/penpot/issue/12543)
 
 ## 2.10.1
 
@@ -78,12 +230,10 @@
 
 - Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
 
-
 ### :bug: Bugs fixed
 
 - Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
 
-
 ## 2.10.0
 
 ### :rocket: Epics and highlights
@@ -99,7 +249,7 @@
 - Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
 - Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
 - Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
-- New Number token - unitless values  [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
+- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
 - New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
 - New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
 - New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
@@ -180,7 +330,6 @@
 - Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
 - Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
 
-
 ### :bug: Bugs fixed
 
 - Copying font size does not copy the unit [Taiga #11143](https://tree.taiga.io/project/penpot/issue/11143)

backend/deps.edn

@@ -28,8 +28,8 @@
   com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.6"
+  {:git/tag "v11.8"
-   :git/sha "94dc017"
+   :git/sha "1d1b33f"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 

backend/dev/user.clj

@@ -27,6 +27,7 @@
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]

backend/resources/app/email/feedback/en.html

@@ -8,38 +8,41 @@
   <body>
     <p>
       <strong>Feedback from:</strong><br />
-      {% if profile %}
+      <span>
-        <span>
+        <span>Name: </span>
-          <span>Name: </span>
+        <span><code>{{profile.fullname|abbreviate:25}}</code></span>
-          <span><code>{{profile.fullname|abbreviate:25}}</code></span>
+      </span>
-        </span>
+      <br />
-        <br />
+      <span>
-
+        <span>Email: </span>
-        <span>
+        <span>{{profile.email}}</span>
-          <span>Email: </span>
+      </span>
-          <span>{{profile.email}}</span>
+      <br />
-        </span>
+      <span>
-        <br />
+        <span>ID: </span>
-
+        <span><code>{{profile.id}}</code></span>
-        <span>
+      </span>
-          <span>ID: </span>
-          <span><code>{{profile.id}}</code></span>
-        </span>
-      {% else %}
-        <span>
-          <span>Email: </span>
-          <span>{{profile.email}}</span>
-        </span>
-      {% endif %}
     </p>
     <p>
       <strong>Subject:</strong><br />
-      <span>{{subject|abbreviate:300}}</span>
+      <span>{{feedback-subject|abbreviate:300}}</span>
     </p>
 
+    <p>
+      <strong>Type:</strong><br />
+      <span>{{feedback-type|abbreviate:300}}</span>
+    </p>
+
+    {% if feedback-error-href %}
+    <p>
+      <strong>Error HREF:</strong><br />
+      <span>{{feedback-error-href|abbreviate:500}}</span>
+    </p>
+    {% endif %}
+
     <p>
       <strong>Message:</strong><br />
-      {{content|linebreaks-br|safe}}
+      {{feedback-content|linebreaks-br}}
     </p>
   </body>
 </html>

backend/resources/app/email/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{subject}}
+[PENPOT FEEDBACK]: {{feedback-subject}}

backend/resources/app/email/feedback/en.txt

@@ -1,9 +1,11 @@
-{% if profile %}
+From: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
-Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+Subject: {{feedback-subject}}
-{% else %}
+Type: {{feedback-type}}
-Feedback from: {{email}}
-{% endif %}
 
-Subject: {{subject}}
+{% if feedback-error-href %}
+HREF: {{feedback-error-href}}
+{% endif -%}
 
-{{content}}
+Message:
+
+{{feedback-content}}

backend/resources/app/email/invite-to-team/en.html

@@ -240,4 +240,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/app/onboarding.edn

@@ -3,7 +3,7 @@
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"}
  {:id "penpot-design-system"
   :name "Penpot Design System | Pencil"
-  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/penpot-app.penpot"}
+  :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Pencil-Penpot-Design-System.penpot"}
  {:id "wireframing-kit"
   :name "Wireframe library"
   :file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}

backend/resources/app/templates/api-doc.tmpl

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="robots" content="noindex,nofollow">
     <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>Builtin API Documentation - Penpot</title>
+    <title>{{label|upper}} API Documentation</title>
 
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
@@ -19,7 +19,7 @@
   <body>
     <main>
       <header>
-        <h1>Penpot API Documentation (v{{version}})</h1>
+        <h1>{{label|upper}}: API Documentation (v{{version}})</h1>
         <small class="menu">
           [
           <nav>
@@ -31,9 +31,10 @@
       </header>
       <section class="doc-content">
         <h2>INTRODUCTION</h2>
-        <p>This documentation is intended to be a general overview of the penpot RPC API.
+        <p>This documentation is intended to be a general overview of
-          If you prefer, you can use <a href="/api/openapi.json">OpenAPI</a>
+          the {{label}} API.  If you prefer, you can
-          and/or <a href="/api/openapi">SwaggerUI</a> as alternative.</p>
+          use <a href="{{openapi}}">Swagger/OpenAPI</a> as
+          alternative.</p>
 
         <h2>GENERAL NOTES</h2>
 
@@ -43,7 +44,7 @@
         that starts with <b>get-</b> in the name, can use GET HTTP
         method which in many cases benefits from the HTTP cache.</p>
 
-
+        {% block auth-section %}
         <h3>Authentication</h3>
         <p>The penpot backend right now offers two way for authenticate the request:
         <b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
@@ -56,9 +57,10 @@
         <p>The access token can be obtained on the appropriate section on profile settings
         and it should be provided using <b>`Authorization`</b> header with <b>`Token
         &lt;token-string&gt;`</b> value.</p>
+        {% endblock %}
 
         <h3>Content Negotiation</h3>
-        <p>The penpot API by default operates indistinctly with: <b>`application/json`</b>
+        <p>This API operates indistinctly with: <b>`application/json`</b>
         and <b>`application/transit+json`</b> content types. You should specify the
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
@@ -75,13 +77,16 @@
         standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
         API</a></p>
 
+        {% block limits-section %}
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share
         the same rate limit). For now the limits are not documented because we are
         studying and analyzing the data. As a general rule, it should not be abused, if an
         abusive use is detected, we will proceed to block the user's access to the
         API.</p>
+        {% endblock %}
 
+        {% block webhooks-section %}
         <h3>Webhooks</h3>
         <p>All methods that emit webhook events are marked with flag <b>WEBHOOK</b>, the
         data structure defined on each method represents the <i>payload</i> of the
@@ -97,9 +102,11 @@
   "profileId": "db601c95-045f-808b-8002-361312e63531"
 }
         </pre>
+        {% endblock %}
+
       </section>
       <section class="rpc-doc-content">
-        <h2>RPC METHODS REFERENCE:</h2>
+        <h2>METHODS REFERENCE:</h2>
         <ul class="rpc-items">
           {% for item in methods %}
             {% include "app/templates/api-doc-entry.tmpl" with item=item %}

backend/resources/app/templates/main-api-doc.tmpl

@@ -0,0 +1 @@
+{% extends "app/templates/api-doc.tmpl" %}

backend/resources/app/templates/management-api-doc.tmpl

@@ -0,0 +1,10 @@
+{% extends "app/templates/api-doc.tmpl" %}
+
+{% block auth-section %}
+{% endblock %}
+
+{% block limits-section %}
+{% endblock %}
+
+{% block webhooks-section %}
+{% endblock %}

backend/resources/app/templates/openapi.tmpl

@@ -7,7 +7,7 @@
     name="description"
     content="SwaggerUI"
   />
-  <title>PENPOT Swagger UI</title>
+  <title>{{label|upper}} API</title>
   <style>{{swagger-css|safe}}</style>
 </head>
 <body>
@@ -16,7 +16,7 @@
 <script>
   window.onload = () => {
     window.ui = SwaggerUIBundle({
-      url: '{{public-uri}}/api/openapi.json',
+      url: '{{uri}}',
       dom_id: '#swagger-ui',      
       presets: [
         SwaggerUIBundle.presets.apis,

backend/resources/log4j2-devenv-repl.xml

@@ -25,8 +25,7 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http" level="info" />
-    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/resources/log4j2-devenv.xml

@@ -25,8 +25,7 @@
     <Logger name="app.storage.tmp" level="info" />
     <Logger name="app.worker" level="trace" />
     <Logger name="app.msgbus" level="info" />
-    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.http" level="info" />
-    <Logger name="app.http.sse" level="info" />
     <Logger name="app.util.websocket" level="info" />
     <Logger name="app.redis" level="info" />
     <Logger name="app.rpc.rlimit" level="info" />

backend/scripts/_env

@@ -1,18 +1,18 @@
 #!/usr/bin/env bash
 
-
+export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key
-export PENPOT_MANAGEMENT_API_SHARED_KEY=super-secret-management-api-key
 export PENPOT_SECRET_KEY=super-secret-devenv-key
 export PENPOT_HOST=devenv
+export PENPOT_PUBLIC_URI=https://localhost:3449
 
 export PENPOT_FLAGS="\
        $PENPOT_FLAGS \
-       enable-login-with-ldap \
        enable-login-with-password
-       enable-login-with-oidc \
+       disable-login-with-ldap \
-       enable-login-with-google \
+       disable-login-with-oidc \
-       enable-login-with-github \
+       disable-login-with-google \
-       enable-login-with-gitlab \
+       disable-login-with-github \
+       disable-login-with-gitlab \
        enable-backend-worker \
        enable-backend-asserts \
        disable-feature-fdata-pointer-map \
@@ -20,6 +20,7 @@ export PENPOT_FLAGS="\
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
+       enable-user-feedback \
        disable-secure-session-cookies \
        enable-smtp \
        enable-prepl-server \
@@ -46,6 +47,8 @@ export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
 # Setup default multipart upload size to 300MiB
 export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
+export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com"
+
 export AWS_ACCESS_KEY_ID=penpot-devenv
 export AWS_SECRET_ACCESS_KEY=penpot-devenv
 export PENPOT_OBJECTS_STORAGE_BACKEND=s3

backend/src/app/binfile/common.clj

@@ -331,6 +331,81 @@
                                  (set/difference cfeat/backend-only-features))
                              #{}))))
 
+(defn check-file-exists
+  [cfg id & {:keys [include-deleted?]
+             :or {include-deleted? false}
+             :as options}]
+  (db/get-with-sql cfg [sql:get-minimal-file id]
+                   {:db/remove-deleted (not include-deleted?)}))
+
+(def ^:private sql:file-permissions
+  "select fpr.is_owner,
+          fpr.is_admin,
+          fpr.can_edit
+     from file_profile_rel as fpr
+    inner join file as f on (f.id = fpr.file_id)
+    where fpr.file_id = ?
+      and fpr.profile_id = ?
+   union all
+   select tpr.is_owner,
+          tpr.is_admin,
+          tpr.can_edit
+     from team_profile_rel as tpr
+    inner join project as p on (p.team_id = tpr.team_id)
+    inner join file as f on (p.id = f.project_id)
+    where f.id = ?
+      and tpr.profile_id = ?
+   union all
+   select ppr.is_owner,
+          ppr.is_admin,
+          ppr.can_edit
+     from project_profile_rel as ppr
+    inner join file as f on (f.project_id = ppr.project_id)
+    where f.id = ?
+      and ppr.profile_id = ?")
+
+(defn- get-file-permissions*
+  [conn profile-id file-id]
+  (when (and profile-id file-id)
+    (db/exec! conn [sql:file-permissions
+                    file-id profile-id
+                    file-id profile-id
+                    file-id profile-id])))
+
+(defn get-file-permissions
+  ([conn profile-id file-id]
+   (let [rows     (get-file-permissions* conn profile-id file-id)
+         is-owner (boolean (some :is-owner rows))
+         is-admin (boolean (some :is-admin rows))
+         can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:type :membership
+        :is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true
+        :is-logged (some? profile-id)})))
+
+  ([conn profile-id file-id share-id]
+   (let [perms  (get-file-permissions conn profile-id file-id)
+         ldata  (some-> (db/get* conn :share-link {:id share-id :file-id file-id})
+                        (dissoc :flags)
+                        (update :pages db/decode-pgarray #{}))]
+
+     ;; NOTE: in a future when share-link becomes more powerful and
+     ;; will allow us specify which parts of the app is available, we
+     ;; will probably need to tweak this function in order to expose
+     ;; this flags to the frontend.
+     (cond
+       (some? perms) perms
+       (some? ldata) {:type :share-link
+                      :can-read true
+                      :pages (:pages ldata)
+                      :is-logged (some? profile-id)
+                      :who-comment (:who-comment ldata)
+                      :who-inspect (:who-inspect ldata)}))))
+
+
 (defn get-project
   [cfg project-id]
   (db/get cfg :project {:id project-id}))
@@ -550,7 +625,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id)]
+              (if-let [library (get-file cfg library-id :include-deleted? true)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data
@@ -749,7 +824,7 @@
           l.version
      FROM libs AS l
     INNER JOIN project AS p ON (p.id = l.project_id)
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+    WHERE l.deleted_at IS NULL;")
 
 (defn get-file-libraries
   [conn file-id]

backend/src/app/binfile/v3.clj

@@ -228,6 +228,7 @@
     (db/tx-run! cfg (fn [cfg]
                       (cond-> (bfc/get-file cfg file-id
                                             {:realize? true
+                                             :include-deleted? true
                                              :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
@@ -254,6 +255,8 @@
 
         (write-entry! output path params)
 
+        (events/tap :progress {:section :storage-object :id id})
+
         (with-open [input (sto/get-object-data storage sobject)]
           (.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
           (io/copy input output :size (:size sobject))
@@ -278,6 +281,8 @@
 
         thumbnails   (bfc/get-file-object-thumbnails cfg file-id)]
 
+    (events/tap :progress {:section :file :id file-id})
+
     (vswap! bfc/*state* update :files assoc file-id
             {:id file-id
              :name (:name file)
@@ -285,14 +290,12 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data))
+                 (assoc :options (:options data)))
 
-                 :always
+          file (-> file
-                 (dissoc :data))
+                   (dissoc :data)
-
+                   (dissoc :deleted-at)
-          file (cond-> file
+                   (encode-file))
-                 :always
-                 (encode-file))
 
           path (str "files/" file-id ".json")]
       (write-entry! output path file))
@@ -818,9 +821,10 @@
         entries (keep (match-storage-entry-fn) entries)]
 
     (doseq [{:keys [id entry]} entries]
-      (let [object  (->> (read-entry input entry)
+      (let [object  (-> (read-entry input entry)
-                         (decode-storage-object)
+                        (decode-storage-object)
-                         (validate-storage-object))
+                        (update :bucket d/nilv sto/default-bucket)
+                        (validate-storage-object))
 
             ext     (cmedia/mtype->extension (:content-type object))
             path    (str "objects/" id ext)

backend/src/app/config.clj

@@ -5,7 +5,6 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.config
-  "A configuration management."
   (:refer-clojure :exclude [get])
   (:require
    [app.common.data :as d]
@@ -47,6 +46,7 @@
    :auto-file-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
+
    :host "localhost"
    :tenant "default"
 
@@ -57,6 +57,8 @@
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
+   :auth-token-cookie-name "auth-token"
+
    :assets-path "/internal/assets/"
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
@@ -90,7 +92,7 @@
     [:secret-key {:optional true} :string]
 
     [:tenant {:optional false} :string]
-    [:public-uri {:optional false} :string]
+    [:public-uri {:optional false} ::sm/uri]
     [:host {:optional false} :string]
 
     [:http-server-port {:optional true} ::sm/int]
@@ -100,7 +102,7 @@
     [:http-server-io-threads {:optional true} ::sm/int]
     [:http-server-max-worker-threads {:optional true} ::sm/int]
 
-    [:management-api-shared-key {:optional true} :string]
+    [:management-api-key {:optional true} :string]
 
     [:telemetry-uri {:optional true} :string]
     [:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
@@ -165,7 +167,7 @@
     [:google-client-id {:optional true} :string]
     [:google-client-secret {:optional true} :string]
     [:oidc-client-id {:optional true} :string]
-    [:oidc-user-info-source {:optional true} :keyword]
+    [:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
     [:oidc-client-secret {:optional true} :string]
     [:oidc-base-uri {:optional true} :string]
     [:oidc-token-uri {:optional true} :string]
@@ -319,5 +321,9 @@
   ([key default]
    (c/get config key default)))
 
+(defn logging-context
+  []
+  {:version/backend (:full version)})
+
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -704,6 +704,12 @@
   (and (sql-exception? cause)
        (= "40001" (.getSQLState ^java.sql.SQLException cause))))
 
+(defn duplicate-key-error?
+  [cause]
+  (and (sql-exception? cause)
+       (= "23505" (.getSQLState ^java.sql.SQLException cause))))
+
+
 (extend-protocol jdbc.prepare/SettableParameter
   clojure.lang.Keyword
   (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]

backend/src/app/email.clj

@@ -7,6 +7,7 @@
 (ns app.email
   "Main api for send emails."
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -93,36 +94,44 @@
                headers)))
 
 (defn- assign-body
-  [^MimeMessage mmsg {:keys [body charset] :or {charset "utf-8"}}]
+  [^MimeMessage mmsg {:keys [body charset attachments] :or {charset "utf-8"}}]
-  (let [mpart (MimeMultipart. "mixed")]
+  (let [mixed-mpart  (MimeMultipart. "mixed")]
     (cond
       (string? body)
-      (let [bpart (MimeBodyPart.)]
+      (let [text-part (MimeBodyPart.)]
-        (.setContent bpart ^String body (str "text/plain; charset=" charset))
+        (.setText text-part ^String body ^String charset)
-        (.addBodyPart mpart bpart))
+        (.addBodyPart mixed-mpart text-part))
-
-      (vector? body)
-      (let [mmp (MimeMultipart. "alternative")
-            mbp (MimeBodyPart.)]
-        (.addBodyPart mpart mbp)
-        (.setContent mbp mmp)
-        (doseq [item body]
-          (let [mbp (MimeBodyPart.)]
-            (.setContent mbp
-                         ^String (:content item)
-                         ^String (str (:type item "text/plain") "; charset=" charset))
-            (.addBodyPart mmp mbp))))
 
       (map? body)
-      (let [bpart (MimeBodyPart.)]
+      (let [content-part      (MimeBodyPart.)
-        (.setContent bpart
+            alternative-mpart (MimeMultipart. "alternative")]
-                     ^String (:content body)
+
-                     ^String (str (:type body "text/plain") "; charset=" charset))
+        (when-let [content (get body "text/plain")]
-        (.addBodyPart mpart bpart))
+          (let [text-part (MimeBodyPart.)]
+            (.setText text-part ^String content ^String charset)
+            (.addBodyPart alternative-mpart text-part)))
+
+        (when-let [content (get body "text/html")]
+          (let [html-part (MimeBodyPart.)]
+            (.setContent html-part ^String content
+                         (str "text/html; charset=" charset))
+            (.addBodyPart alternative-mpart html-part)))
+
+        (.setContent content-part alternative-mpart)
+        (.addBodyPart mixed-mpart content-part))
 
       :else
-      (throw (ex-info "Unsupported type" {:body body})))
+      (throw (IllegalArgumentException. "invalid email body provided")))
-    (.setContent mmsg mpart)
+
+    (doseq [[name content] attachments]
+
+      (prn "attachment" name)
+      (let [attachment-part (MimeBodyPart.)]
+        (.setFileName attachment-part ^String name)
+        (.setContent attachment-part ^String content (str "text/plain; charset=" charset))
+        (.addBodyPart mixed-mpart attachment-part)))
+
+    (.setContent mmsg mixed-mpart)
     mmsg))
 
 (defn- opts->props
@@ -210,24 +219,26 @@
       (ex/raise :type :internal
                 :code :missing-email-templates))
     {:subject subj
-     :body (into
+     :body (d/without-nils
-            [{:type "text/plain"
+            {"text/plain" text
-              :content text}]
+             "text/html" html})}))
-            (when html
-              [{:type "text/html"
-                :content html}]))}))
 
-(def ^:private schema:context
+(def ^:private schema:params
-  [:map
+  [:map {:title "Email Params"}
    [:to [:or ::sm/email [::sm/vec ::sm/email]]]
    [:reply-to {:optional true} ::sm/email]
    [:from {:optional true} ::sm/email]
    [:lang {:optional true} ::sm/text]
+   [:subject {:optional true} ::sm/text]
    [:priority {:optional true} [:enum :high :low]]
-   [:extra-data {:optional true} ::sm/text]])
+   [:extra-data {:optional true} ::sm/text]
+   [:body {:optional true}
+    [:or :string [:map-of :string :string]]]
+   [:attachments {:optional true}
+    [:map-of :string :string]]])
 
-(def ^:private check-context
+(def ^:private check-params
-  (sm/check-fn schema:context))
+  (sm/check-fn schema:params))
 
 (defn template-factory
   [& {:keys [id schema]}]
@@ -235,9 +246,9 @@
   (let [check-fn (if schema
                    (sm/check-fn schema)
                    (constantly nil))]
-    (fn [context]
+    (fn [params]
-      (let [context (-> context check-context check-fn)
+      (let [params (-> params check-params check-fn)
-            email   (build-email-template id context)]
+            email  (build-email-template id params)]
         (when-not email
           (ex/raise :type :internal
                     :code :email-template-does-not-exists
@@ -245,35 +256,40 @@
                     :template-id id))
 
         (cond-> (assoc email :id (name id))
-          (:extra-data context)
+          (:extra-data params)
-          (assoc :extra-data (:extra-data context))
+          (assoc :extra-data (:extra-data params))
 
-          (:from context)
+          (seq (:attachments params))
-          (assoc :from (:from context))
+          (assoc :attachments (:attachments params))
 
-          (:reply-to context)
+          (:from params)
-          (assoc :reply-to (:reply-to context))
+          (assoc :from (:from params))
 
-          (:to context)
+          (:reply-to params)
-          (assoc :to (:to context)))))))
+          (assoc :reply-to (:reply-to params))
+
+          (:to params)
+          (assoc :to (:to params)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PUBLIC HIGH-LEVEL API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn render
-  [email-factory context]
+  [email-factory params]
-  (email-factory context))
+  (email-factory params))
 
 (defn send!
   "Schedule an already defined email to be sent using asynchronously
   using worker task."
-  [{:keys [::conn ::factory] :as context}]
+  [{:keys [::conn ::factory] :as params}]
   (assert (db/connectable? conn) "expected a valid database connection or pool")
 
   (let [email (if factory
-                (factory context)
+                (factory params)
-                (dissoc context ::conn))]
+                (-> params
+                    (dissoc params)
+                    (check-params)))]
     (wrk/submit! {::wrk/task :sendmail
                   ::wrk/delay 0
                   ::wrk/max-retries 4
@@ -343,8 +359,10 @@
 
 (def ^:private schema:feedback
   [:map
-   [:subject ::sm/text]
+   [:feedback-subject ::sm/text]
-   [:content ::sm/text]])
+   [:feedback-type ::sm/text]
+   [:feedback-content ::sm/text]
+   [:profile :map]])
 
 (def user-feedback
   "A profile feedback email."

backend/src/app/http.clj

@@ -25,7 +25,6 @@
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [integrant.core :as ig]
    [reitit.core :as r]
@@ -149,7 +148,6 @@
   [:map
    [::ws/routes schema:routes]
    [::rpc/routes schema:routes]
-   [::rpc.doc/routes schema:routes]
    [::oidc/routes schema:routes]
    [::assets/routes schema:routes]
    [::debug/routes schema:routes]
@@ -171,8 +169,9 @@
                       [sec/sec-fetch-metadata]
                       [mw/params]
                       [mw/format-response]
-                      [session/soft-auth cfg]
+                      [mw/auth {:bearer (partial session/decode-token cfg)
-                      [actoken/soft-auth cfg]
+                                :cookie (partial session/decode-token cfg)
+                                :token  (partial actoken/decode-token cfg)}]
                       [mw/parse-request]
                       [mw/errors errors/handle]
                       [mw/restrict-methods]]}
@@ -188,9 +187,5 @@
       (::mgmt/routes cfg)]
 
      (::ws/routes cfg)
-
+     (::oidc/routes cfg)
-     ["/api" {:middleware [[mw/cors]
+     (::rpc/routes cfg)]]))
-                           [sec/client-header-check]]}
-      (::oidc/routes cfg)
-      (::rpc.doc/routes cfg)
-      (::rpc/routes cfg)]]]))

backend/src/app/http/access_token.clj

@@ -9,23 +9,19 @@
    [app.common.logging :as l]
    [app.config :as cf]
    [app.db :as db]
+   [app.http :as-alias http]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
-   [app.tokens :as tokens]
+   [app.tokens :as tokens]))
-   [yetti.request :as yreq]))
 
-(def header-re #"(?i)^Token\s+(.*)")
+(defn decode-token
-
-(defn get-token
-  [request]
-  (some->> (yreq/get-header request "authorization")
-           (re-matches header-re)
-           (second)))
-
-(defn- decode-token
   [cfg token]
-  (when token
+  (try
-    (tokens/verify cfg {:token token :iss "access-token"})))
+    (tokens/verify cfg {:token token :iss "access-token"})
+    (catch Throwable cause
+      (l/trc :hint "exception on decoding token"
+             :token token
+             :cause cause))))
 
 (def sql:get-token-data
   "SELECT perms, profile_id, expires_at
@@ -35,47 +31,28 @@
            OR (expires_at > now()));")
 
 (defn- get-token-data
-  [pool token-id]
+  [pool claims]
   (when-not (db/read-only? pool)
-    (some-> (db/exec-one! pool [sql:get-token-data token-id])
+    (when-let [token-id (get claims :tid)]
-            (update :perms db/decode-pgarray #{}))))
+      (some-> (db/exec-one! pool [sql:get-token-data token-id])
-
+              (update :perms db/decode-pgarray #{})))))
-(defn- wrap-soft-auth
-  "Soft Authentication, will be executed synchronously on the undertow
-  worker thread."
-  [handler cfg]
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token cfg token)]
-                (cond-> request
-                  (map? claims)
-                  (assoc ::id (:tid claims))))
-              (catch Throwable cause
-                (l/trace :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
 
 (defn- wrap-authz
-  "Authorization middleware, will be executed synchronously on vthread."
   [handler {:keys [::db/pool]}]
   (fn [request]
-    (let [{:keys [perms profile-id expires-at]} (some->> (::id request) (get-token-data pool))]
+    (let [{:keys [type claims]} (get request ::http/auth-data)]
-      (handler (cond-> request
+      (if (= :token type)
-                 (some? perms)
+        (let [{:keys [perms profile-id expires-at]} (some->> claims (get-token-data pool))]
-                 (assoc ::perms perms)
+          ;; FIXME: revisit this, this data looks unused
-                 (some? profile-id)
+          (handler (cond-> request
-                 (assoc ::profile-id profile-id)
+                     (some? perms)
-                 (some? expires-at)
+                     (assoc ::perms perms)
-                 (assoc ::expires-at expires-at))))))
+                     (some? profile-id)
+                     (assoc ::profile-id profile-id)
+                     (some? expires-at)
+                     (assoc ::expires-at expires-at))))
 
-(def soft-auth
+        (handler request)))))
-  {:name ::soft-auth
-   :compile (fn [& _]
-              (when (contains? cf/flags :access-tokens)
-                wrap-soft-auth))})
 
 (def authz
   {:name ::authz

backend/src/app/http/assets.clj

@@ -30,7 +30,7 @@
 
 (defn- get-file-media-object
   [pool id]
-  (db/get pool :file-media-object {:id id}))
+  (db/get pool :file-media-object {:id id} {::db/remove-deleted false}))
 
 (defn- serve-object-from-s3
   [{:keys [::sto/storage] :as cfg} obj]

backend/src/app/http/client.clj

@@ -9,8 +9,7 @@
   (:require
    [app.common.schema :as sm]
    [integrant.core :as ig]
-   [java-http-clj.core :as http]
+   [java-http-clj.core :as http])
-   [promesa.core :as p])
   (:import
    java.net.http.HttpClient))
 
@@ -29,14 +28,9 @@
 
 (defn send!
   ([client req] (send! client req {}))
-  ([client req {:keys [response-type sync?] :or {response-type :string sync? false}}]
+  ([client req {:keys [response-type] :or {response-type :string}}]
    (assert (client? client) "expected valid http client")
-   (if sync?
+   (http/send req {:client client :as response-type})))
-     (http/send req {:client client :as response-type})
-     (try
-       (http/send-async req {:client client :as response-type})
-       (catch Throwable cause
-         (p/rejected cause))))))
 
 (defn- resolve-client
   [params]
@@ -56,8 +50,8 @@
   ([cfg-or-client request]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request {:sync? true})))
+     (send! client request {})))
   ([cfg-or-client request options]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request (merge {:sync? true} options)))))
+     (send! client request options))))

backend/src/app/http/errors.clj

@@ -13,6 +13,7 @@
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
+   [app.http.auth :as-alias auth]
    [app.http.session :as-alias session]
    [app.util.inet :as inet]
    [clojure.spec.alpha :as s]
@@ -22,18 +23,16 @@
 (defn request->context
   "Extracts error report relevant context data from request."
   [request]
-  (let [claims (-> {}
+  (let [{:keys [claims] :as auth} (get request ::http/auth-data)]
-                   (into (::session/token-claims request))
+    (-> (cf/logging-context)
-                   (into (::actoken/token-claims request)))]
+        (assoc :request/path (:path request))
-    {:request/path       (:path request)
+        (assoc :request/method (:method request))
-     :request/method     (:method request)
+        (assoc :request/params (:params request))
-     :request/params     (:params request)
+        (assoc :request/user-agent (yreq/get-header request "user-agent"))
-     :request/user-agent (yreq/get-header request "user-agent")
+        (assoc :request/ip-addr (inet/parse-request request))
-     :request/ip-addr    (inet/parse-request request)
+        (assoc :request/profile-id (get claims :uid))
-     :request/profile-id (:uid claims)
+        (assoc :request/auth-data auth)
-     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
+        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
-     :version/backend    (:full cf/version)}))
-
 
 (defmulti handle-error
   (fn [cause _ _]
@@ -61,7 +60,6 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
-        (l/wrn :hint "restriction error" :cause err)
         {::yres/status 400
          ::yres/body data}))))
 

backend/src/app/http/management.clj

@@ -13,7 +13,7 @@
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
-   [app.http.access-token :refer [get-token]]
+   [app.http.middleware :as mw]
    [app.main :as-alias main]
    [app.rpc.commands.profile :as cmd.profile]
    [app.setup :as-alias setup]
@@ -32,20 +32,6 @@
   [_ params]
   (assert (db/pool? (::db/pool params)) "expect valid database pool"))
 
-(def ^:private auth
-  {:name ::auth
-   :compile
-   (fn [_ _]
-     (fn [handler shared-key]
-       (if shared-key
-         (fn [request]
-           (let [token (get-token request)]
-             (if (= token shared-key)
-               (handler request)
-               {::yres/status 403})))
-         (fn [_ _]
-           {::yres/status 403}))))})
-
 (def ^:private default-system
   {:name ::default-system
    :compile
@@ -64,23 +50,27 @@
            (db/tx-run! cfg handler request)))))})
 
 (defmethod ig/init-key ::routes
-  [_ cfg]
+  [_ {:keys [::setup/props] :as cfg}]
-  ["" {:middleware [[auth (cf/get :management-api-shared-key)]
-                    [default-system cfg]
-                    [transaction]]}
-   ["/authenticate"
-    {:handler authenticate
-     :allowed-methods #{:post}}]
 
-   ["/get-customer"
+  (let [management-key (or (cf/get :management-api-key)
-    {:handler get-customer
+                           (get props :management-key))]
-     :transaction true
-     :allowed-methods #{:post}}]
 
-   ["/update-customer"
+    ["" {:middleware [[mw/shared-key-auth management-key]
-    {:handler update-customer
+                      [default-system cfg]
-     :allowed-methods #{:post}
+                      [transaction]]}
-     :transaction true}]])
+     ["/authenticate"
+      {:handler authenticate
+       :allowed-methods #{:post}}]
+
+     ["/get-customer"
+      {:handler get-customer
+       :transaction true
+       :allowed-methods #{:post}}]
+
+     ["/update-customer"
+      {:handler update-customer
+       :allowed-methods #{:post}
+       :transaction true}]]))
 
 ;; ---- HELPERS
 

backend/src/app/http/middleware.clj

@@ -12,8 +12,11 @@
    [app.common.schema :as-alias sm]
    [app.common.transit :as t]
    [app.config :as cf]
+   [app.http :as-alias http]
    [app.http.errors :as errors]
+   [app.tokens :as tokens]
    [app.util.pointer-map :as pmap]
+   [buddy.core.codecs :as bc]
    [cuerdas.core :as str]
    [yetti.adapter :as yt]
    [yetti.middleware :as ymw]
@@ -240,3 +243,77 @@
              (if (contains? allowed method)
                (handler request)
                {::yres/status 405}))))))})
+
+(defn- wrap-auth
+  [handler decoders]
+  (let [token-re
+        #"(?i)^(Token|Bearer)\s+(.*)"
+
+        get-token-from-authorization
+        (fn [request]
+          (when-let [[_ token-type token] (some->> (yreq/get-header request "authorization")
+                                                   (re-matches token-re))]
+            (if (= "token" (str/lower token-type))
+              {:type :token
+               :token token}
+              {:type :bearer
+               :token token})))
+
+        get-token-from-cookie
+        (fn [request]
+          (let [cname (cf/get :auth-token-cookie-name)
+                token (some-> (yreq/get-cookie request cname) :value)]
+            (when-not (str/empty? token)
+              {:type :cookie
+               :token token})))
+
+        get-token
+        (some-fn get-token-from-cookie get-token-from-authorization)
+
+        process-request
+        (fn [request]
+          (if-let [{:keys [type token] :as auth} (get-token request)]
+            (let [decode-fn (get decoders type)]
+              (if (or (= type :cookie) (= type :bearer))
+                (let [metadata (tokens/decode-header token)]
+                  ;; NOTE: we only proceed to decode claims on new
+                  ;; cookie tokens. The old cookies dont need to be
+                  ;; decoded because they use the token string as ID
+                  (if (and (= (:kid metadata) 1)
+                           (= (:ver metadata) 1)
+                           (some? decode-fn))
+                    (assoc request ::http/auth-data (assoc auth
+                                                           :claims (decode-fn token)
+                                                           :metadata metadata))
+                    (assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
+
+                (if decode-fn
+                  (assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
+                  (assoc request ::http/auth-data auth))))
+
+            request))]
+
+    (fn [request]
+      (-> request process-request handler))))
+
+(def auth
+  {:name ::auth
+   :compile (constantly wrap-auth)})
+
+(defn- wrap-shared-key-auth
+  [handler shared-key]
+  (if shared-key
+    (let [shared-key (if (string? shared-key)
+                       shared-key
+                       (bc/bytes->b64-str shared-key true))]
+      (fn [request]
+        (let [key (yreq/get-header request "x-shared-key")]
+          (if (= key shared-key)
+            (handler (assoc request ::http/auth-with-shared-key true))
+            {::yres/status 403}))))
+    (fn [_ _]
+      {::yres/status 403})))
+
+(def shared-key-auth
+  {:name ::shared-key-auth
+   :compile (constantly wrap-shared-key-auth)})

backend/src/app/http/session.clj

@@ -11,28 +11,24 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.http :as-alias http]
+   [app.http.auth :as-alias http.auth]
    [app.http.session.tasks :as-alias tasks]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [cuerdas.core :as str]
    [integrant.core :as ig]
-   [yetti.request :as yreq]))
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; A default cookie name for storing the session.
-(def default-auth-token-cookie-name "auth-token")
-
-;; A cookie that we can use to check from other sites of the same
-;; domain if a user is authenticated.
-(def default-auth-data-cookie-name "auth-data")
-
 ;; Default value for cookie max-age
 (def default-cookie-max-age (ct/duration {:days 7}))
 
@@ -44,10 +40,10 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol ISessionManager
-  (read [_ key])
+  (read-session [_ id])
-  (write! [_ key data])
+  (create-session [_ params])
-  (update! [_ data])
+  (update-session [_ session])
-  (delete! [_ key]))
+  (delete-session [_ id]))
 
 (defn manager?
   [o]
@@ -62,71 +58,82 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private schema:params
-  [:map {:title "session-params"}
+  [:map {:title "SessionParams" :closed true}
-   [:user-agent ::sm/text]
    [:profile-id ::sm/uuid]
-   [:created-at ::ct/inst]])
+   [:user-agent {:optional true} ::sm/text]
+   [:sso-provider-id {:optional true} ::sm/uuid]
+   [:sso-session-id {:optional true} :string]])
 
 (def ^:private valid-params?
   (sm/validator schema:params))
 
-(defn- prepare-session-params
-  [params key]
-  (assert (string? key) "expected key to be a string")
-  (assert (not (str/blank? key)) "expected key to be not empty")
-  (assert (valid-params? params) "expected valid params")
-
-  {:user-agent (:user-agent params)
-   :profile-id (:profile-id params)
-   :created-at (:created-at params)
-   :updated-at (:created-at params)
-   :id key})
-
 (defn- database-manager
   [pool]
   (reify ISessionManager
-    (read [_ token]
+    (read-session [_ id]
-      (db/exec-one! pool (sql/select :http-session {:id token})))
+      (if (string? id)
+        ;; Backward compatibility
+        (let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
+          (-> session
+              (assoc :modified-at (:updated-at session))
+              (dissoc :updated-at)))
+        (db/exec-one! pool (sql/select :http-session-v2 {:id id}))))
 
-    (write! [_ key params]
+    (create-session [_ params]
-      (let [params (-> params
+      (assert (valid-params? params) "expect valid session params")
-                       (assoc :created-at (ct/now))
-                       (prepare-session-params key))]
-        (db/insert! pool :http-session params)
-        params))
 
-    (update! [_ params]
+      (let [now    (ct/now)
-      (let [updated-at (ct/now)]
+            params (-> params
-        (db/update! pool :http-session
+                       (assoc :id (uuid/next))
-                    {:updated-at updated-at}
+                       (assoc :created-at now)
-                    {:id (:id params)})
+                       (assoc :modified-at now))]
-        (assoc params :updated-at updated-at)))
+        (db/insert! pool :http-session-v2 params
+                    {::db/return-keys true})))
 
-    (delete! [_ token]
+    (update-session [_ session]
-      (db/delete! pool :http-session {:id token})
+      (let [modified-at (ct/now)]
+        (if (string? (:id session))
+          (db/insert! pool :http-session-v2
+                      (-> session
+                          (assoc :id (uuid/next))
+                          (assoc :created-at modified-at)
+                          (assoc :modified-at modified-at)))
+          (db/update! pool :http-session-v2
+                      {:modified-at modified-at}
+                      {:id (:id session)}
+                      {::db/return-keys true}))))
+
+    (delete-session [_ id]
+      (if (string? id)
+        (db/delete! pool :http-session {:id id} {::db/return-keys false})
+        (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
       nil)))
 
 (defn inmemory-manager
   []
   (let [cache (atom {})]
     (reify ISessionManager
-      (read [_ token]
+      (read-session [_ id]
-        (get @cache token))
+        (get @cache id))
 
-      (write! [_ key params]
+      (create-session [_ params]
-        (let [params (-> params
+        (assert (valid-params? params) "expect valid session params")
-                         (assoc :created-at (ct/now))
-                         (prepare-session-params key))]
-          (swap! cache assoc key params)
-          params))
 
-      (update! [_ params]
+        (let [now     (ct/now)
-        (let [updated-at (ct/now)]
+              session (-> params
-          (swap! cache update (:id params) assoc :updated-at updated-at)
+                          (assoc :id (uuid/next))
-          (assoc params :updated-at updated-at)))
+                          (assoc :created-at now)
+                          (assoc :modified-at now))]
+          (swap! cache assoc (:id session) session)
+          session))
 
-      (delete! [_ token]
+      (update-session [_ session]
-        (swap! cache dissoc token)
+        (let [modified-at (ct/now)]
+          (swap! cache update (:id session) assoc :modified-at modified-at)
+          (assoc session :modified-at modified-at)))
+
+      (delete-session [_ id]
+        (swap! cache dissoc id)
         nil))))
 
 (defmethod ig/assert-key ::manager
@@ -146,103 +153,116 @@
 ;; MANAGER IMPL
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare ^:private assign-auth-token-cookie)
+(declare ^:private assign-session-cookie)
-(declare ^:private clear-auth-token-cookie)
+(declare ^:private clear-session-cookie)
-(declare ^:private gen-token)
+
+(defn- assign-token
+  [cfg session]
+  (let [claims {:iss "authentication"
+                :aud "penpot"
+                :sid (:id session)
+                :iat (:modified-at session)
+                :uid (:profile-id session)
+                :sso-provider-id (:sso-provider-id session)
+                :sso-session-id (:sso-session-id session)}
+        header {:kid 1 :ver 1}
+        token  (tokens/generate cfg claims header)]
+    (assoc session :token token)))
 
 (defn create-fn
-  [{:keys [::manager] :as cfg} profile-id]
+  [{:keys [::manager] :as cfg} {profile-id :id :as profile}
+   & {:keys [sso-provider-id sso-session-id]}]
+
   (assert (manager? manager) "expected valid session manager")
   (assert (uuid? profile-id) "expected valid uuid for profile-id")
 
   (fn [request response]
     (let [uagent  (yreq/get-header request "user-agent")
-          params  {:profile-id profile-id
+          session (->> {:user-agent uagent
-                   :user-agent uagent}
+                        :profile-id profile-id
-          token   (gen-token cfg params)
+                        :sso-provider-id sso-provider-id
-          session (write! manager token params)]
+                        :sso-session-id sso-session-id}
-      (l/trc :hint "create" :profile-id (str profile-id))
+                       (d/without-nils)
-      (-> response
+                       (create-session manager)
-          (assign-auth-token-cookie session)))))
+                       (assign-token cfg))]
+
+      (l/trc :hint "create" :id (str (:id session)) :profile-id (str profile-id))
+      (assign-session-cookie response session))))
 
 (defn delete-fn
   [{:keys [::manager]}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request response]
-    (let [cname   (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+    (some->> (get request ::id) (delete-session manager))
-          cookie  (yreq/get-cookie request cname)]
+    (clear-session-cookie response)))
-      (l/trc :hint "delete" :profile-id (:profile-id request))
-      (some->> (:value cookie) (delete! manager))
-      (-> response
-          (assoc :status 204)
-          (assoc :body nil)
-          (clear-auth-token-cookie)))))
 
-(defn- gen-token
+(defn decode-token
-  [cfg {:keys [profile-id created-at]}]
-  (tokens/generate cfg {:iss "authentication"
-                        :iat created-at
-                        :uid profile-id}))
-(defn- decode-token
   [cfg token]
-  (when token
+  (try
-    (tokens/verify cfg {:token token :iss "authentication"})))
+    (tokens/verify cfg {:token token :iss "authentication"})
+    (catch Throwable cause
+      (l/trc :hint "exception on decoding token"
+             :token token
+             :cause cause))))
 
-(defn- get-token
+(defn get-session
   [request]
-  (let [cname  (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+  (get request ::session))
-        cookie (some-> (yreq/get-cookie request cname) :value)]
-    (when-not (str/empty? cookie)
-      cookie)))
 
-(defn- get-session
+(defn invalidate-others
-  [manager token]
+  [cfg session]
-  (some->> token (read manager)))
+  (let [sql "delete from http_session_v2 where profile_id = ? and id != ?"]
+    (-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
+        (db/get-update-count))))
 
 (defn- renew-session?
-  [{:keys [updated-at] :as session}]
+  [{:keys [id modified-at] :as session}]
-  (and (ct/inst? updated-at)
+  (or (string? id)
-       (let [elapsed (ct/diff updated-at (ct/now))]
+      (and (ct/inst? modified-at)
-         (neg? (compare default-renewal-max-age elapsed)))))
+           (let [elapsed (ct/diff modified-at (ct/now))]
-
+             (neg? (compare default-renewal-max-age elapsed))))))
-(defn- wrap-soft-auth
-  [handler {:keys [::manager] :as cfg}]
-  (assert (manager? manager) "expected valid session manager")
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token cfg token)]
-                (cond-> request
-                  (map? claims)
-                  (-> (assoc ::token-claims claims)
-                      (assoc ::token token))))
-              (catch Throwable cause
-                (l/trc :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
 
 (defn- wrap-authz
-  [handler {:keys [::manager]}]
+  [handler {:keys [::manager] :as cfg}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request]
-    (let [session  (get-session manager (::token request))
+    (let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
-          request  (cond-> request
+      (cond
-                     (some? session)
+        (= type :cookie)
-                     (assoc ::profile-id (:profile-id session)
+        (let [session (case (:ver metadata)
-                            ::id (:id session)))
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
-          response (handler request)]
+                        0 (read-session manager token)
+                        1 (some->> (:sid claims) (read-session manager))
+                        nil)
 
-      (if (renew-session? session)
+              request (cond-> request
-        (let [session (update! manager session)]
+                        (some? session)
-          (-> response
+                        (-> (assoc ::profile-id (:profile-id session))
-              (assign-auth-token-cookie session)))
+                            (assoc ::session session)))
-        response))))
 
-(def soft-auth
+              response (handler request)]
-  {:name ::soft-auth
+
-   :compile (constantly wrap-soft-auth)})
+          (if (and session (renew-session? session))
+            (let [session (->> session
+                               (update-session manager)
+                               (assign-token cfg))]
+              (assign-session-cookie response session))
+            response))
+
+        (= type :bearer)
+        (let [session (case (:ver metadata)
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
+                        0 (read-session manager token)
+                        1 (some->> (:sid claims) (read-session manager))
+                        nil)
+              request (cond-> request
+                        (some? session)
+                        (-> (assoc ::profile-id (:profile-id session))
+                            (assoc ::session session)))]
+          (handler request))
+
+        :else
+        (handler request)))))
 
 (def authz
   {:name ::authz
@@ -250,16 +270,16 @@
 
 ;; --- IMPL
 
-(defn- assign-auth-token-cookie
+(defn- assign-session-cookie
-  [response {token :id updated-at :updated-at}]
+  [response {token :token modified-at :modified-at}]
   (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        created-at updated-at
+        created-at modified-at
         renewal    (ct/plus created-at default-renewal-max-age)
         expires    (ct/plus created-at max-age)
         secure?    (contains? cf/flags :secure-session-cookies)
         strict?    (contains? cf/flags :strict-session-cookies)
         cors?      (contains? cf/flags :cors)
-        name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+        name       (cf/get :auth-token-cookie-name)
         comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
         cookie     {:path "/"
                     :http-only true
@@ -268,12 +288,12 @@
                     :comment comment
                     :same-site (if cors? :none (if strict? :strict :lax))
                     :secure secure?}]
-    (update response :cookies assoc name cookie)))
+    (update response ::yres/cookies assoc name cookie)))
 
-(defn- clear-auth-token-cookie
+(defn- clear-session-cookie
   [response]
-  (let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
+  (let [cname (cf/get :auth-token-cookie-name)]
-    (update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
+    (update response ::yres/cookies assoc cname {:path "/" :value "" :max-age 0})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASK: SESSION GC

backend/src/app/loggers/audit.clj

@@ -25,7 +25,8 @@
    [app.util.inet :as inet]
    [app.util.services :as-alias sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [yetti.request :as yreq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -78,17 +79,32 @@
          (remove #(contains? reserved-props (key %))))
         props))
 
-(defn event-from-rpc-params
+(defn get-external-session-id
-  "Create a base event skeleton with pre-filled some important
+  [request]
-  data that can be extracted from RPC params object"
+  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-  [params]
+    (when-not (or (> (count session-id) 256)
-  (let [context {:external-session-id (::rpc/external-session-id params)
+                  (= session-id "null")
-                 :external-event-origin (::rpc/external-event-origin params)
+                  (str/blank? session-id))
-                 :triggered-by (::rpc/handler-name params)}]
+      session-id)))
-    {::type "action"
+
-     ::profile-id (::rpc/profile-id params)
+(defn- get-client-event-origin
-     ::ip-addr (::rpc/ip-addr params)
+  [request]
-     ::context (d/without-nils context)}))
+  (when-let [origin (yreq/get-header request "x-event-origin")]
+    (when-not (or (= origin "null")
+                  (str/blank? origin))
+      (str/prune origin 200))))
+
+(defn get-client-user-agent
+  [request]
+  (when-let [user-agent (yreq/get-header request "user-agent")]
+    (str/prune user-agent 500)))
+
+(defn- get-client-version
+  [request]
+  (when-let [origin (yreq/get-header request "x-frontend-version")]
+    (when-not (or (= origin "null")
+                  (str/blank? origin))
+      (str/prune origin 100))))
 
 ;; --- SPECS
 
@@ -117,6 +133,33 @@
 (def ^:private check-event
   (sm/check-fn schema:event))
 
+(defn- prepare-context-from-request
+  [request]
+  (let [client-event-origin (get-client-event-origin request)
+        client-version      (get-client-version request)
+        client-user-agent   (get-client-user-agent request)
+        session-id          (get-external-session-id request)
+        token-id       (::actoken/id request)]
+    (d/without-nils
+     {:external-session-id session-id
+      :access-token-id (some-> token-id str)
+      :client-event-origin client-event-origin
+      :client-user-agent client-user-agent
+      :client-version client-version
+      :version (:full cf/version)})))
+
+(defn event-from-rpc-params
+  "Create a base event skeleton with pre-filled some important
+  data that can be extracted from RPC params object"
+  [params]
+  (let [context (some-> params meta ::http/request prepare-context-from-request)
+        event   {::type "action"
+                 ::profile-id (or (::rpc/profile-id params) uuid/zero)
+                 ::ip-addr (::rpc/ip-addr params)}]
+    (cond-> event
+      (some? context)
+      (assoc ::context context))))
+
 (defn prepare-event
   [cfg mdata params result]
   (let [resultm      (meta result)
@@ -126,23 +169,15 @@
                          (::rpc/profile-id params)
                          uuid/zero)
 
-        session-id   (get params ::rpc/external-session-id)
-        event-origin (get params ::rpc/external-event-origin)
         props        (-> (or (::replace-props resultm)
                              (-> params
                                  (merge (::props resultm))
                                  (dissoc :profile-id)
                                  (dissoc :type)))
-
                          (clean-props))
 
-        token-id     (::actoken/id request)
+        context      (merge (::context resultm)
-        context      (-> (::context resultm)
+                            (prepare-context-from-request request))
-                         (assoc :external-session-id session-id)
-                         (assoc :external-event-origin event-origin)
-                         (assoc :access-token-id (some-> token-id str))
-                         (d/without-nils))
-
         ip-addr      (inet/parse-request request)]
 
     {::type (or (::type resultm)

backend/src/app/loggers/audit/archive_task.clj

@@ -57,7 +57,7 @@
                                   :uid uuid/zero})
         body    (t/encode {:events events})
         headers {"content-type" "application/transit+json"
-                 "origin" (cf/get :public-uri)
+                 "origin" (str (cf/get :public-uri))
                  "cookie" (u/map->query-string {:auth-token token})}
         params  {:uri uri
                  :timeout 12000

backend/src/app/loggers/database.clj

@@ -49,7 +49,7 @@
           ctx  (-> context
                    (assoc :tenant (cf/get :tenant))
                    (assoc :host (cf/get :host))
-                   (assoc :public-uri (cf/get :public-uri))
+                   (assoc :public-uri (str (cf/get :public-uri)))
                    (assoc :logger/name logger)
                    (assoc :logger/level level)
                    (dissoc :request/params :value :params :data))]

backend/src/app/main.clj

@@ -21,7 +21,7 @@
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
    [app.http.management :as mgmt]
-   [app.http.session :as-alias session]
+   [app.http.session :as session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
    [app.loggers.webhooks :as-alias webhooks]
@@ -31,7 +31,6 @@
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [app.srepl :as-alias srepl]
    [app.storage :as-alias sto]
@@ -260,14 +259,17 @@
    ::oidc.providers/generic
    {::http.client/client (ig/ref ::http.client/client)}
 
+   ::oidc/providers
+   [(ig/ref ::oidc.providers/google)
+    (ig/ref ::oidc.providers/github)
+    (ig/ref ::oidc.providers/gitlab)
+    (ig/ref ::oidc.providers/generic)]
+
    ::oidc/routes
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
     ::setup/props        (ig/ref ::setup/props)
-    ::oidc/providers     {:google (ig/ref ::oidc.providers/google)
+    ::oidc/providers     (ig/ref ::oidc/providers)
-                          :github (ig/ref ::oidc.providers/github)
-                          :gitlab (ig/ref ::oidc.providers/gitlab)
-                          :oidc   (ig/ref ::oidc.providers/generic)}
     ::session/manager    (ig/ref ::session/manager)
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
@@ -280,7 +282,6 @@
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
     ::rpc/routes         (ig/ref ::rpc/routes)
-    ::rpc.doc/routes     (ig/ref ::rpc.doc/routes)
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
@@ -300,6 +301,7 @@
    {::db/pool         (ig/ref ::db/pool)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::mbus/msgbus     (ig/ref ::mbus/msgbus)
+    ::setup/props     (ig/ref ::setup/props)
     ::session/manager (ig/ref ::session/manager)}
 
    :app.http.assets/routes
@@ -337,14 +339,26 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   :app.rpc.doc/routes
+   :app.rpc/management-methods
-   {:app.rpc/methods (ig/ref :app.rpc/methods)}
+   {::http.client/client (ig/ref ::http.client/client)
+    ::db/pool            (ig/ref ::db/pool)
+    ::rds/pool           (ig/ref ::rds/pool)
+    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::session/manager    (ig/ref ::session/manager)
+    ::sto/storage        (ig/ref ::sto/storage)
+    ::mtx/metrics        (ig/ref ::mtx/metrics)
+    ::mbus/msgbus        (ig/ref ::mbus/msgbus)
+    ::rds/client         (ig/ref ::rds/client)
+    ::setup/props        (ig/ref ::setup/props)}
 
    ::rpc/routes
-   {::rpc/methods     (ig/ref :app.rpc/methods)
+   {::rpc/methods        (ig/ref :app.rpc/methods)
-    ::db/pool         (ig/ref ::db/pool)
+    ::rpc/management-methods (ig/ref :app.rpc/management-methods)
-    ::session/manager (ig/ref ::session/manager)
+
-    ::setup/props     (ig/ref ::setup/props)}
+    ;; FIXME: revisit if db/pool is necessary here
+    ::db/pool                (ig/ref ::db/pool)
+    ::session/manager        (ig/ref ::session/manager)
+    ::setup/props            (ig/ref ::setup/props)}
 
    ::wrk/registry
    {::mtx/metrics (ig/ref ::mtx/metrics)

backend/src/app/media.clj

@@ -17,6 +17,7 @@
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
+   [app.http.client :as http]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [buddy.core.bytes :as bb]
@@ -37,6 +38,9 @@
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
+(def default-max-file-size
+  (* 1024 1024 10)) ; 10 MiB
+
 (def schema:upload
   [:map {:title "Upload"}
    [:filename :string]
@@ -241,7 +245,7 @@
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (merge input info {:ts (ct/now)}))
+        (merge input info {:ts (ct/now) :size (fs/size path)}))
 
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
@@ -261,6 +265,7 @@
           (assoc input
                  :width  width
                  :height height
+                 :size (fs/size path)
                  :ts (ct/now)))))))
 
 (defmethod process-error org.im4java.core.InfoException
@@ -270,6 +275,54 @@
             :hint "invalid image"
             :cause error))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMAGE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn download-image
+  "Download an image from the provided URI and return the media input object"
+  [{:keys [::http/client]} uri]
+  (letfn [(parse-and-validate [{:keys [headers] :as response}]
+            (let [size     (some-> (get headers "content-length") d/parse-integer)
+                  mtype    (get headers "content-type")
+                  format   (cm/mtype->format mtype)
+                  max-size (cf/get :media-max-file-size default-max-file-size)]
+
+              (when-not size
+                (ex/raise :type :validation
+                          :code :unknown-size
+                          :hint "seems like the url points to resource with unknown size"))
+
+              (when (> size max-size)
+                (ex/raise :type :validation
+                          :code :file-too-large
+                          :hint (str/ffmt "the file size % is greater than the maximum %"
+                                          size
+                                          default-max-file-size)))
+
+              (when (nil? format)
+                (ex/raise :type :validation
+                          :code :media-type-not-allowed
+                          :hint "seems like the url points to an invalid media object"))
+
+              {:size size :mtype mtype :format format}))]
+
+    (let [{:keys [body] :as response} (http/req! client
+                                                 {:method :get :uri uri}
+                                                 {:response-type :input-stream})
+          {:keys [size mtype]} (parse-and-validate response)
+          path    (tmp/tempfile :prefix "penpot.media.download.")
+          written (io/write* path body :size size)]
+
+      (when (not= written size)
+        (ex/raise :type :internal
+                  :code :mismatch-write-size
+                  :hint "unexpected state: unable to write to file"))
+
+      {;; :size size
+       :path path
+       :mtype mtype})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FONTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/migrations.clj

@@ -450,7 +450,13 @@
     :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
 
    {:name "0141-add-file-data-table.sql"
-    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}
+
+   {:name "0142-add-sso-provider-table"
+    :fn (mg/resource "app/migrations/sql/0142-add-sso-provider-table.sql")}
+
+   {:name "0143-http-session-v2-table"
+    :fn (mg/resource "app/migrations/sql/0143-add-http-session-v2-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0142-add-sso-provider-table.sql

@@ -0,0 +1,33 @@
+CREATE TABLE sso_provider (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  is_enabled boolean NOT NULL DEFAULT true,
+
+  type text NOT NULL CHECK (type IN ('oidc')),
+  domain text NOT NULL,
+
+  client_id text NOT NULL,
+  client_secret text NOT NULL,
+
+  base_uri text NOT NULL,
+  token_uri text NULL,
+  auth_uri text NULL,
+  user_uri text NULL,
+  jwks_uri text NULL,
+  logout_uri text NULL,
+
+  roles_attr text NULL,
+  email_attr text NULL,
+  name_attr text NULL,
+  user_info_source text NOT NULL DEFAULT 'token'
+     CHECK (user_info_source IN ('token', 'userinfo', 'auto')),
+
+  scopes text[] NULL,
+  roles text[] NULL
+);
+
+CREATE UNIQUE INDEX sso_provider__domain__idx
+    ON sso_provider(domain);

backend/src/app/migrations/sql/0143-add-http-session-v2-table.sql

@@ -0,0 +1,23 @@
+CREATE TABLE http_session_v2 (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  profile_id uuid REFERENCES profile(id) ON DELETE CASCADE,
+  user_agent text NULL,
+
+  sso_provider_id uuid NULL REFERENCES sso_provider(id) ON DELETE CASCADE,
+  sso_session_id text NULL
+);
+
+CREATE INDEX http_session_v2__profile_id__idx
+    ON http_session_v2(profile_id);
+
+CREATE INDEX http_session_v2__sso_provider_id__idx
+    ON http_session_v2(sso_provider_id)
+ WHERE sso_provider_id IS NOT NULL;
+
+CREATE INDEX http_session_v2__sso_session_id__idx
+    ON http_session_v2(sso_session_id)
+ WHERE sso_session_id IS NOT NULL;

backend/src/app/rpc.clj

@@ -13,11 +13,15 @@
    [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.time :as ct]
+   [app.common.uri :as u]
+   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
    [app.http.access-token :as actoken]
    [app.http.client :as-alias http.client]
+   [app.http.middleware :as mw]
+   [app.http.security :as sec]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
@@ -26,6 +30,7 @@
    [app.redis :as rds]
    [app.rpc.climit :as climit]
    [app.rpc.cond :as cond]
+   [app.rpc.doc :as doc]
    [app.rpc.helpers :as rph]
    [app.rpc.retry :as retry]
    [app.rpc.rlimit :as rlimit]
@@ -36,7 +41,6 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [promesa.core :as p]
    [yetti.request :as yreq]
    [yetti.response :as yres]))
 
@@ -44,7 +48,7 @@
 
 (defn- default-handler
   [_]
-  (p/rejected (ex/error :type :not-found)))
+  (ex/raise :type :not-found))
 
 (defn- handle-response-transformation
   [response request mdata]
@@ -64,67 +68,62 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result (rph/unwrap result)]
+                   (let [result  (rph/unwrap result)
-                     {::yres/status  (::http/status mdata 200)
+                         status  (or (::http/status mdata)
-                      ::yres/headers (::http/headers mdata {})
+                                     (if (nil? result)
+                                       204
+                                       200))
+                         headers (cond-> (::http/headers mdata {})
+                                   (yres/stream-body? result)
+                                   (assoc "content-type" "application/octet-stream"))]
+                     {::yres/status  status
+                      ::yres/headers headers
                       ::yres/body    result}))]
+
     (-> response
         (handle-response-transformation request mdata)
         (handle-before-comple-hook mdata))))
 
-(defn get-external-session-id
+(defn- make-rpc-handler
-  [request]
-  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-    (when-not (or (> (count session-id) 256)
-                  (= session-id "null")
-                  (str/blank? session-id))
-      session-id)))
-
-(defn- get-external-event-origin
-  [request]
-  (when-let [origin (yreq/get-header request "x-event-origin")]
-    (when-not (or (> (count origin) 256)
-                  (= origin "null")
-                  (str/blank? origin))
-      origin)))
-
-(defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
-  [methods {:keys [params path-params method] :as request}]
+  [methods]
-  (let [handler-name (:type path-params)
+  (let [methods (update-vals methods peek)]
-        etag         (yreq/get-header request "if-none-match")
+    (fn [{:keys [params path-params method] :as request}]
-        profile-id   (or (::session/profile-id request)
+      (let [handler-name (:type path-params)
-                         (::actoken/profile-id request))
+            etag         (yreq/get-header request "if-none-match")
+            profile-id   (or (::session/profile-id request)
+                             (::actoken/profile-id request)
+                             (if (::http/auth-with-shared-key request)
+                               uuid/zero
+                               nil))
 
-        ip-addr      (inet/parse-request request)
+            ip-addr      (inet/parse-request request)
-        session-id   (get-external-session-id request)
-        event-origin (get-external-event-origin request)
 
-        data         (-> params
+            data         (-> params
-                         (assoc ::handler-name handler-name)
+                             (assoc ::handler-name handler-name)
-                         (assoc ::ip-addr ip-addr)
+                             (assoc ::ip-addr ip-addr)
-                         (assoc ::request-at (ct/now))
+                             (assoc ::request-at (ct/now))
-                         (assoc ::external-session-id session-id)
+                             (assoc ::cond/key etag)
-                         (assoc ::external-event-origin event-origin)
+                             (cond-> (uuid? profile-id)
-                         (assoc ::session/id (::session/id request))
+                               (assoc ::profile-id profile-id)))
-                         (assoc ::cond/key etag)
-                         (cond-> (uuid? profile-id)
-                           (assoc ::profile-id profile-id)))
 
-        data         (vary-meta data assoc ::http/request request)
+            data         (with-meta data
-        handler-fn   (get methods (keyword handler-name) default-handler)]
+                           {::http/request request})
 
-    (when (and (or (= method :get)
+            handler-fn   (get methods (keyword handler-name) default-handler)]
-                   (= method :head))
-               (not (str/starts-with? handler-name "get-")))
-      (ex/raise :type :restriction
-                :code :method-not-allowed
-                :hint "method not allowed for this request"))
 
-    (binding [cond/*enabled* true]
+        (when (and (or (= method :get)
-      (let [response (handler-fn data)]
+                       (= method :head))
-        (handle-response request response)))))
+                   (not (str/starts-with? handler-name "get-")))
+          (ex/raise :type :restriction
+                    :code :method-not-allowed
+                    :hint "method not allowed for this request"))
+
+        ;; FIXME: why we have this cond enabled here, we need to move it outside this handler
+        (binding [cond/*enabled* true]
+          (let [response (handler-fn data)]
+            (handle-response request response)))))))
 
 (defn- wrap-metrics
   "Wrap service method with metrics measurement."
@@ -201,7 +200,7 @@
                         ::sm/explain (explain params)))))))
     f))
 
-(defn- wrap-all
+(defn- wrap
   [cfg f mdata]
   (as-> f $
     (wrap-db-transaction cfg $ mdata)
@@ -215,17 +214,30 @@
     (wrap-params-validation cfg $ mdata)
     (wrap-authentication cfg $ mdata)))
 
-(defn- wrap
+(defn- wrap-management
   [cfg f mdata]
-  (l/trc :hint "register method" :name (::sv/name mdata))
+  (as-> f $
-  (let [f (wrap-all cfg f mdata)]
+    (wrap-db-transaction cfg $ mdata)
-    (partial f cfg)))
+    (retry/wrap-retry cfg $ mdata)
+    (climit/wrap cfg $ mdata)
+    (wrap-metrics cfg $ mdata)
+    (wrap-audit cfg $ mdata)
+    (wrap-spec-conform cfg $ mdata)
+    (wrap-params-validation cfg $ mdata)
+    (wrap-authentication cfg $ mdata)))
 
 (defn- process-method
-  [cfg [vfn mdata]]
+  [cfg module wrap-fn [f mdata]]
-  [(keyword (::sv/name mdata)) [mdata (wrap cfg vfn mdata)]])
+  (l/trc :hint "add method" :module module :name (::sv/name mdata))
+  (let [f (wrap-fn cfg f mdata)
+        k (keyword (::sv/name mdata))]
+    [k [mdata (partial f cfg)]]))
 
-(defn- resolve-command-methods
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; API METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-methods
   [cfg]
   (let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
     (->> (sv/scan-ns
@@ -254,7 +266,7 @@
           'app.rpc.commands.verify-token
           'app.rpc.commands.viewer
           'app.rpc.commands.webhooks)
-         (map (partial process-method cfg))
+         (map (partial process-method cfg "rpc" wrap))
          (into {}))))
 
 (def ^:private schema:methods-params
@@ -278,7 +290,50 @@
 (defmethod ig/init-key ::methods
   [_ cfg]
   (let [cfg (d/without-nils cfg)]
-    (resolve-command-methods cfg)))
+    (resolve-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MANAGEMENT METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-management-methods
+  [cfg]
+  (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
+    (->> (sv/scan-ns
+          'app.rpc.management.subscription
+          'app.rpc.management.exporter)
+         (map (partial process-method cfg "management" wrap-management))
+         (into {}))))
+
+(def ^:private schema:management-methods-params
+  [:map {:title "management-methods-params"}
+   ::session/manager
+   ::http.client/client
+   ::db/pool
+   ::rds/pool
+   ::mbus/msgbus
+   ::sto/storage
+   ::mtx/metrics
+   ::setup/props])
+
+(defmethod ig/assert-key ::management-methods
+  [_ params]
+  (assert (sm/check schema:management-methods-params params)))
+
+(defmethod ig/init-key ::management-methods
+  [_ cfg]
+  (let [cfg (d/without-nils cfg)]
+    (resolve-management-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ROUTES
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- redirect
+  [href]
+  (fn [_]
+    {::yres/status 308
+     ::yres/headers {"location" (str href)}}))
 
 (def ^:private schema:methods
   [:map-of :keyword [:tuple :map ::sm/fn]])
@@ -293,11 +348,50 @@
   (assert (db/pool? (::db/pool params)) "expect valid database pool")
   (assert (some? (::setup/props params)))
   (assert (session/manager? (::session/manager params)) "expect valid session manager")
-  (assert (valid-methods? (::methods params)) "expect valid methods map"))
+  (assert (valid-methods? (::methods params)) "expect valid methods map")
+  (assert (valid-methods? (::management-methods params)) "expect valid methods map"))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [::methods] :as cfg}]
+  [_ {:keys [::methods ::management-methods ::setup/props] :as cfg}]
-  (let [methods (update-vals methods peek)]
+
-    [["/rpc" {:middleware [[session/authz cfg]
+  (let [public-uri     (cf/get :public-uri)
-                           [actoken/authz cfg]]}
+        management-key (or (cf/get :management-api-key)
-      ["/command/:type" {:handler (partial rpc-handler methods)}]]]))
+                           (get props :management-key))]
+
+    ["/api"
+     ["/management"
+      ["/methods/:type"
+       {:middleware [[mw/shared-key-auth management-key]
+                     [session/authz cfg]]
+        :handler (make-rpc-handler management-methods)}]
+
+      (doc/routes :methods management-methods
+                  :label "management"
+                  :base-uri (u/join public-uri "/api/management")
+                  :description "MANAGEMENT API")]
+
+     ["/main"
+      ["/methods/:type"
+       {:middleware [[mw/cors]
+                     [sec/client-header-check]
+                     [session/authz cfg]
+                     [actoken/authz cfg]]
+        :handler (make-rpc-handler methods)}]
+
+      (doc/routes :methods methods
+                  :label "main"
+                  :base-uri (u/join public-uri "/api/main")
+                  :description "MAIN API")]
+
+     ;; BACKWARD COMPATIBILITY
+     ["/_doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/openapi" {:handler (redirect (u/join public-uri "/api/main/doc/openapi"))}]
+     ["/openapi.join" {:handler (redirect (u/join public-uri "/api/main/doc/openapi.json"))}]
+
+     ["/rpc/command/:type"
+      {:middleware [[mw/cors]
+                    [sec/client-header-check]
+                    [session/authz cfg]
+                    [actoken/authz cfg]]
+       :handler (make-rpc-handler methods)}]]))

backend/src/app/rpc/commands/access_token.clj

@@ -28,6 +28,7 @@
         expires-at (some-> expiration (ct/in-future))
         created-at (ct/now)
         token      (tokens/generate cfg {:iss "access-token"
+                                         :uid profile-id
                                          :iat created-at
                                          :tid token-id})
 

backend/src/app/rpc/commands/auth.clj

@@ -7,21 +7,24 @@
 (ns app.rpc.commands.auth
   (:require
    [app.auth :as auth]
+   [app.auth.oidc :as oidc]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.email :as eml]
    [app.email.blacklist :as email.blacklist]
    [app.email.whitelist :as email.whitelist]
+   [app.http :as-alias http]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
+   [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
    [app.rpc.commands.profile :as profile]
@@ -30,6 +33,7 @@
    [app.rpc.helpers :as rph]
    [app.setup :as-alias setup]
    [app.setup.welcome-file :refer [create-welcome-file]]
+   [app.storage :as sto]
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.worker :as wrk]
@@ -109,7 +113,7 @@
                                (assoc profile :is-admin (let [admins (cf/get :admins)]
                                                           (contains? admins (:email profile)))))]
               (-> response
-                  (rph/with-transform (session/create-fn cfg (:id profile)))
+                  (rph/with-transform (session/create-fn cfg profile))
                   (rph/with-meta {::audit/props (audit/profile->props profile)
                                   ::audit/profile-id (:id profile)}))))]
 
@@ -145,7 +149,24 @@
   [cfg params]
   (if (= (:profile-id params)
          (::rpc/profile-id params))
-    (rph/with-transform {} (session/delete-fn cfg))
+    (let [{:keys [claims]}
+          (rph/get-auth-data params)
+
+          provider
+          (some->> (get claims :sso-provider-id)
+                   (oidc/get-provider cfg))
+
+          response
+          (if (and provider (:logout-uri provider))
+            (let [params {"logout_hint" (get claims :sso-session-id)
+                          "client_id" (get provider :client-id)
+                          "post_logout_redirect_uri" (str (cf/get :public-uri))}
+                  uri    (-> (u/uri (:logout-uri provider))
+                             (assoc :query (u/map->query-string params)))]
+              {:redirect-uri uri})
+            {})]
+
+      (rph/with-transform response (session/delete-fn cfg)))
     {}))
 
 ;; ---- COMMAND: Recover Profile
@@ -271,11 +292,30 @@
 
 ;; ---- COMMAND: Register Profile
 
-(defn create-profile!
+(defn import-profile-picture
+  [cfg uri]
+  (try
+    (let [storage (sto/resolve cfg)
+          input   (media/download-image cfg uri)
+          input   (media/run {:cmd :info :input input})
+          hash    (sto/calculate-hash (:path input))
+          content (-> (sto/content (:path input) (:size input))
+                      (sto/wrap-with-hash hash))
+          sobject (sto/put-object! storage {::sto/content content
+                                            ::sto/deduplicate? true
+                                            :bucket "profile"
+                                            :content-type (:mtype input)})]
+      (:id sobject))
+    (catch Throwable cause
+      (l/wrn :hint "unable to import profile picture"
+             :uri uri
+             :cause cause)
+      nil)))
+
+(defn create-profile
   "Create the profile entry on the database with limited set of input
   attrs (all the other attrs are filled with default values)."
-  [conn {:keys [email] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
-  (dm/assert! ::sm/email email)
   (let [id        (or (:id params) (uuid/next))
         props     (-> (audit/extract-utm-params params)
                       (merge (:props params))
@@ -283,8 +323,7 @@
                               :viewed-walkthrough? false
                               :nudge {:big 10 :small 1}
                               :v2-info-shown true
-                              :release-notes-viewed (:main cf/version)})
+                              :release-notes-viewed (:main cf/version)}))
-                      (db/tjson))
 
         password  (or (:password params) "!")
 
@@ -299,6 +338,12 @@
         theme     (:theme params nil)
         email     (str/lower email)
 
+        photo-id  (some->> (or (:oidc/picture props)
+                               (:google/picture props)
+                               (:github/picture props)
+                               (:gitlab/picture props))
+                           (import-profile-picture cfg))
+
         params    {:id id
                    :fullname (:fullname params)
                    :email email
@@ -306,27 +351,26 @@
                    :lang locale
                    :password password
                    :deleted-at (:deleted-at params)
-                   :props props
+                   :props (db/tjson props)
                    :theme theme
+                   :photo-id photo-id
                    :is-active is-active
                    :is-muted is-muted
                    :is-demo is-demo}]
+
     (try
       (-> (db/insert! conn :profile params)
           (profile/decode-row))
       (catch org.postgresql.util.PSQLException cause
-        (let [state (.getSQLState cause)]
+        (if (db/duplicate-key-error? cause)
-          (if (not= state "23505")
+          (ex/raise :type :validation
-            (throw cause)
+                    :code :email-already-exists
+                    :hint "email already exists"
+                    :cause cause)
+          (throw cause))))))
 
-            (do
-              (l/error :hint "not an error" :cause cause)
-              (ex/raise :type :validation
-                        :code :email-already-exists
-                        :hint "email already exists"
-                        :cause cause))))))))
 
-(defn create-profile-rels!
+(defn create-profile-rels
   [conn {:keys [id] :as profile}]
   (let [features (cfeat/get-enabled-features cf/flags)
         team     (teams/create-team conn
@@ -376,12 +420,13 @@
                      ;; to detect if the profile is already registered
                      (or (profile/get-profile-by-email conn (:email claims))
                          (let [is-active (or (boolean (:is-active claims))
+                                             (boolean (:email-verified claims))
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
                                              (update :password auth/derive-password))
-                               profile   (->> (create-profile! conn params)
+                               profile   (->> (create-profile cfg params)
-                                              (create-profile-rels! conn))]
+                                              (create-profile-rels conn))]
                            (vary-meta profile assoc :created true))))
 
         created?   (-> profile meta :created true?)
@@ -419,10 +464,10 @@
       (and (some? invitation)
            (= (:email profile)
               (:member-email invitation)))
-      (let [claims (assoc invitation :member-id  (:id profile))
+      (let [invitation (assoc invitation :member-id  (:id profile))
-            token  (tokens/generate cfg claims)]
+            token      (tokens/generate cfg invitation)]
         (-> {:invitation-token token}
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-meta {::audit/replace-props props
                             ::audit/context {:action "accept-invitation"}
                             ::audit/profile-id (:id profile)})))
@@ -433,7 +478,7 @@
       created?
       (if (:is-active profile)
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-defer create-welcome-file-when-needed)
             (rph/with-meta
               {::audit/replace-props props
@@ -562,4 +607,32 @@
   [cfg params]
   (db/tx-run! cfg request-profile-recovery params))
 
+;; --- COMMAND: get-sso-config
 
+(defn- extract-domain
+  "Extract the domain part from email"
+  [email]
+  (let [at (str/last-index-of email "@")]
+    (when (and (>= at 0)
+               (< at (dec (count email))))
+      (-> (subs email (inc at))
+          (str/trim)
+          (str/lower)))))
+
+(def ^:private schema:get-sso-provider
+  [:map {:title "get-sso-config"}
+   [:email ::sm/email]])
+
+(def ^:private schema:get-sso-provider-result
+  [:map {:title "SSOProvider"}
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::get-sso-provider
+  {::rpc/auth false
+   ::doc/added "2.12"
+   ::sm/params schema:get-sso-provider
+   ::sm/result schema:get-sso-provider-result}
+  [cfg {:keys [email]}]
+  (when-let [domain (extract-domain email)]
+    (when-let [config (db/get* cfg :sso-provider {:domain domain})]
+      (select-keys config [:id]))))

backend/src/app/rpc/commands/binfile.clj

@@ -11,9 +11,9 @@
    [app.binfile.v1 :as bf.v1]
    [app.binfile.v3 :as bf.v3]
    [app.common.features :as cfeat]
-   [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.http.sse :as sse]
@@ -25,10 +25,12 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
+   [app.storage :as sto]
+   [app.storage.tmp :as tmp]
    [app.tasks.file-gc]
    [app.util.services :as sv]
    [app.worker :as-alias wrk]
-   [yetti.response :as yres]))
+   [datoteka.fs :as fs]))
 
 (set! *warn-on-reflection* true)
 
@@ -38,57 +40,42 @@
   schema:export-binfile
   [:map {:title "export-binfile"}
    [:file-id ::sm/uuid]
-   [:version {:optional true} ::sm/int]
    [:include-libraries ::sm/boolean]
    [:embed-assets ::sm/boolean]])
 
-(defn stream-export-v1
+(defn- export-binfile
-  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+  [{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
-  (yres/stream-body
+  (let [output  (tmp/tempfile*)]
-   (fn [_ output-stream]
+    (try
-     (try
+      (-> cfg
-       (-> cfg
+          (assoc ::bfc/ids #{file-id})
-           (assoc ::bfc/ids #{file-id})
+          (assoc ::bfc/embed-assets embed-assets)
-           (assoc ::bfc/embed-assets embed-assets)
+          (assoc ::bfc/include-libraries include-libraries)
-           (assoc ::bfc/include-libraries include-libraries)
+          (bf.v3/export-files! output))
-           (bf.v1/export-files! output-stream))
-       (catch Throwable cause
-         (l/err :hint "exception on exporting file"
-                :file-id (str file-id)
-                :cause cause))))))
 
-(defn stream-export-v3
+      (let [data   (sto/content output)
-  [cfg {:keys [file-id include-libraries embed-assets] :as params}]
+            object (sto/put-object! storage
-  (yres/stream-body
+                                    {::sto/content data
-   (fn [_ output-stream]
+                                     ::sto/touched-at (ct/in-future {:minutes 60})
-     (try
+                                     :content-type "application/zip"
-       (-> cfg
+                                     :bucket "tempfile"})]
-           (assoc ::bfc/ids #{file-id})
+
-           (assoc ::bfc/embed-assets embed-assets)
+        (-> (cf/get :public-uri)
-           (assoc ::bfc/include-libraries include-libraries)
+            (u/join "/assets/by-id/")
-           (bf.v3/export-files! output-stream))
+            (u/join (str (:id object)))))
-       (catch Throwable cause
+
-         (l/err :hint "exception on exporting file"
+      (finally
-                :file-id (str file-id)
+        (fs/delete output)))))
-                :cause cause))))))
 
 (sv/defmethod ::export-binfile
   "Export a penpot file in a binary format."
   {::doc/added "1.15"
+   ::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
    ::webhooks/event? true
    ::sm/params schema:export-binfile}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (fn [_]
+  (sse/response (partial export-binfile cfg params)))
-    (let [version (or version 1)
-          body    (case (int version)
-                    1 (stream-export-v1 cfg params)
-                    2 (throw (ex-info "not-implemented" {}))
-                    3 (stream-export-v3 cfg params))]
-
-      {::yres/status 200
-       ::yres/headers {"content-type" "application/octet-stream"}
-       ::yres/body body})))
 
 ;; --- Command: import-binfile
 

backend/src/app/rpc/commands/comments.clj

@@ -234,36 +234,39 @@
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
-(def ^:private sql:comment-threads
+(defn- get-comment-threads-sql
-  "SELECT DISTINCT ON (ct.id)
+  [where]
-          ct.*,
+  (str/ffmt
-          pf.fullname AS owner_fullname,
+   "SELECT DISTINCT ON (ct.id)
-          pf.email AS owner_email,
+           ct.*,
-          pf.photo_id AS owner_photo_id,
+           pf.fullname AS owner_fullname,
-          p.team_id AS team_id,
+           pf.email AS owner_email,
-          f.name AS file_name,
+           pf.photo_id AS owner_photo_id,
-          f.project_id AS project_id,
+           p.team_id AS team_id,
-          first_value(c.content) OVER w AS content,
+           f.name AS file_name,
-          (SELECT count(1)
+           f.project_id AS project_id,
-             FROM comment AS c
+           first_value(c.content) OVER w AS content,
-            WHERE c.thread_id = ct.id) AS count_comments,
+           (SELECT count(1)
-          (SELECT count(1)
+              FROM comment AS c
-             FROM comment AS c
+             WHERE c.thread_id = ct.id) AS count_comments,
-            WHERE c.thread_id = ct.id
+           (SELECT count(1)
-              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+              FROM comment AS c
-     FROM comment_thread AS ct
+             WHERE c.thread_id = ct.id
-    INNER JOIN comment AS c ON (c.thread_id = ct.id)
+               AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
-    INNER JOIN file AS f ON (f.id = ct.file_id)
+      FROM comment_thread AS ct
-    INNER JOIN project AS p ON (p.id = f.project_id)
+     INNER JOIN comment AS c ON (c.thread_id = ct.id)
-     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+     INNER JOIN file AS f ON (f.id = ct.file_id)
-     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+     INNER JOIN project AS p ON (p.id = f.project_id)
-    WHERE f.deleted_at IS NULL
+      LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
-      AND p.deleted_at IS NULL
+      LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
-   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
+     WHERE f.deleted_at IS NULL
+       AND p.deleted_at IS NULL
+       %1
+    WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)"
+   where))
 
 (def ^:private sql:comment-threads-by-file-id
-  (str "WITH threads AS (" sql:comment-threads ")"
+  (get-comment-threads-sql "AND ct.file_id = ?"))
-       "SELECT * FROM threads WHERE file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
@@ -273,34 +276,29 @@
 ;; --- COMMAND: Get Unread Comment Threads
 
 (def ^:private sql:unread-all-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
+  (str "WITH threads AS ("
-       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
+       (get-comment-threads-sql "AND p.team_id = ?")
+       ")"
+       "SELECT t.* FROM threads AS t
+         WHERE t.count_unread_comments > 0"))
 
-;; The partial configuration will retrieve only comments created by the user and
-;; threads that have a mention to the user.
 (def ^:private sql:unread-partial-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
+  (str "WITH threads AS ("
-       "SELECT * FROM threads
+       (get-comment-threads-sql "AND p.team_id = ? AND (ct.owner_id = ? OR ? = ANY(ct.mentions))")
-         WHERE count_unread_comments > 0
+       ")"
-           AND team_id = ?
+       "SELECT t.* FROM threads AS t
-           AND (owner_id = ? OR ? = ANY(mentions))"))
+         WHERE t.count_unread_comments > 0"))
 
 (defn- get-unread-comment-threads
   [cfg profile-id team-id]
-  (let [profile (-> (db/get cfg :profile {:id profile-id})
+  (let [profile (-> (db/get cfg :profile {:id profile-id} ::db/remove-deleted false)
                     (profile/decode-row))
-        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)
-
+        result  (case notify
-    (case notify
+                  :all     (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
-      :all
+                  :partial (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-      (->> (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
+                  [])]
-           (into [] xf-decode-row))
+    (into [] xf-decode-row result)))
-
-      :partial
-      (->> (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-           (into [] xf-decode-row))
-
-      [])))
 
 (def ^:private
   schema:get-unread-comment-threads
@@ -323,16 +321,17 @@
    [:id ::sm/uuid]
    [:share-id {:optional true} [:maybe ::sm/uuid]]])
 
+(def ^:private sql:get-comment-thread
+  (get-comment-threads-sql "AND ct.file_id = ? AND ct.id = ?"))
+
 (sv/defmethod ::get-comment-thread
   {::doc/added "1.15"
    ::sm/params schema:get-comment-thread}
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
+                 (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
-                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
+                         (decode-row)))))
-                   (-> (db/exec-one! conn [sql profile-id id file-id])
-                       (decode-row))))))
 
 ;; --- COMMAND: Retrieve Comments
 

backend/src/app/rpc/commands/demo.clj

@@ -39,18 +39,19 @@
         fullname (str "Demo User " sem)
 
         password (-> (bn/random-bytes 16)
-                     (bc/bytes->b64u)
+                     (bc/bytes->b64 true)
                      (bc/bytes->str))
 
         params   {:email email
                   :fullname fullname
                   :is-active true
+                  :is-demo true
                   :deleted-at (ct/in-future (cf/get-deletion-delay))
                   :password (derive-password password)
                   :props {}}
-        profile  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
+        profile  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
-                                   (->> (auth/create-profile! conn params)
+                                   (->> (auth/create-profile cfg params)
-                                        (auth/create-profile-rels! conn))))]
+                                        (auth/create-profile-rels conn))))]
     (with-meta {:email email
                 :password password}
       {::audit/profile-id (:id profile)})))

backend/src/app/rpc/commands/feedback.clj

@@ -7,6 +7,7 @@
 (ns app.rpc.commands.feedback
   "A general purpose feedback module."
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.config :as cf]
@@ -21,8 +22,11 @@
 
 (def ^:private schema:send-user-feedback
   [:map {:title "send-user-feedback"}
-   [:subject [:string {:max 400}]]
+   [:subject [:string {:max 500}]]
-   [:content [:string {:max 2500}]]])
+   [:content [:string {:max 2500}]]
+   [:type {:optional true} :string]
+   [:error-href {:optional true} [:string {:max 2500}]]
+   [:error-report {:optional true} :string]])
 
 (sv/defmethod ::send-user-feedback
   {::doc/added "1.18"
@@ -39,16 +43,26 @@
 
 (defn- send-user-feedback!
   [pool profile params]
-  (let [dest (or (cf/get :user-feedback-destination)
+  (let [destination
-                 ;; LEGACY
+        (or (cf/get :user-feedback-destination)
-                 (cf/get :feedback-destination))]
+            ;; LEGACY
+            (cf/get :feedback-destination))
+
+        attachments
+        (d/without-nils
+         {"error-report.txt" (:error-report params)})]
+
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/user-feedback
-                :from     dest
+                :from     (cf/get :smtp-default-from)
-                :to       dest
+                :to       destination
-                :profile  profile
                 :reply-to (:email profile)
                 :email    (:email profile)
-                :subject  (:subject params)
+                :attachments attachments
-                :content  (:content params)})
+
+                :feedback-subject (:subject params)
+                :feedback-type (:type params "not-specified")
+                :feedback-content (:content params)
+                :feedback-error-href (:error-href params)
+                :profile profile})
     nil))

backend/src/app/rpc/commands/files.clj

@@ -26,6 +26,7 @@
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
    [app.features.logical-deletion :as ldel]
+   [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.msgbus :as mbus]
@@ -38,6 +39,7 @@
    [app.rpc.helpers :as rph]
    [app.rpc.permissions :as perms]
    [app.util.blob :as blob]
+   [app.util.events :as events]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.worker :as wrk]
@@ -77,85 +79,14 @@
 
 ;; --- FILE PERMISSIONS
 
-
-(def ^:private sql:file-permissions
-  "select fpr.is_owner,
-          fpr.is_admin,
-          fpr.can_edit
-     from file_profile_rel as fpr
-    inner join file as f on (f.id = fpr.file_id)
-    where fpr.file_id = ?
-      and fpr.profile_id = ?
-      and f.deleted_at is null
-   union all
-   select tpr.is_owner,
-          tpr.is_admin,
-          tpr.can_edit
-     from team_profile_rel as tpr
-    inner join project as p on (p.team_id = tpr.team_id)
-    inner join file as f on (p.id = f.project_id)
-    where f.id = ?
-      and tpr.profile_id = ?
-      and f.deleted_at is null
-   union all
-   select ppr.is_owner,
-          ppr.is_admin,
-          ppr.can_edit
-     from project_profile_rel as ppr
-    inner join file as f on (f.project_id = ppr.project_id)
-    where f.id = ?
-      and ppr.profile_id = ?
-      and f.deleted_at is null")
-
-(defn get-file-permissions
-  [conn profile-id file-id]
-  (when (and profile-id file-id)
-    (db/exec! conn [sql:file-permissions
-                    file-id profile-id
-                    file-id profile-id
-                    file-id profile-id])))
-
-(defn get-permissions
-  ([conn profile-id file-id]
-   (let [rows     (get-file-permissions conn profile-id file-id)
-         is-owner (boolean (some :is-owner rows))
-         is-admin (boolean (some :is-admin rows))
-         can-edit (boolean (some :can-edit rows))]
-     (when (seq rows)
-       {:type :membership
-        :is-owner is-owner
-        :is-admin (or is-owner is-admin)
-        :can-edit (or is-owner is-admin can-edit)
-        :can-read true
-        :is-logged (some? profile-id)})))
-
-  ([conn profile-id file-id share-id]
-   (let [perms  (get-permissions conn profile-id file-id)
-         ldata  (some-> (db/get* conn :share-link {:id share-id :file-id file-id})
-                        (dissoc :flags)
-                        (update :pages db/decode-pgarray #{}))]
-
-     ;; NOTE: in a future when share-link becomes more powerful and
-     ;; will allow us specify which parts of the app is available, we
-     ;; will probably need to tweak this function in order to expose
-     ;; this flags to the frontend.
-     (cond
-       (some? perms) perms
-       (some? ldata) {:type :share-link
-                      :can-read true
-                      :pages (:pages ldata)
-                      :is-logged (some? profile-id)
-                      :who-comment (:who-comment ldata)
-                      :who-inspect (:who-inspect ldata)}))))
-
 (def has-edit-permissions?
-  (perms/make-edition-predicate-fn get-permissions))
+  (perms/make-edition-predicate-fn bfc/get-file-permissions))
 
 (def has-read-permissions?
-  (perms/make-read-predicate-fn get-permissions))
+  (perms/make-read-predicate-fn bfc/get-file-permissions))
 
 (def has-comment-permissions?
-  (perms/make-comment-predicate-fn get-permissions))
+  (perms/make-comment-predicate-fn bfc/get-file-permissions))
 
 (def check-edition-permissions!
   (perms/make-check-fn has-edit-permissions?))
@@ -168,7 +99,7 @@
 
 (defn check-comment-permissions!
   [conn profile-id file-id share-id]
-  (let [perms       (get-permissions conn profile-id file-id share-id)
+  (let [perms       (bfc/get-file-permissions conn profile-id file-id share-id)
         can-read    (has-read-permissions? perms)
         can-comment (has-comment-permissions? perms)]
     (when-not (or can-read can-comment)
@@ -220,7 +151,7 @@
 (defn- get-minimal-file-with-perms
   [cfg {:keys [:id ::rpc/profile-id]}]
   (let [mfile (get-minimal-file cfg id)
-        perms (get-permissions cfg profile-id id)]
+        perms (bfc/get-file-permissions cfg profile-id id)]
     (assoc mfile :permissions perms)))
 
 (defn get-file-etag
@@ -246,7 +177,7 @@
   ;; will be already prefetched and we just reuse them instead
   ;; of making an additional database queries.
   (let [perms (or (:permissions (::cond/object params))
-                  (get-permissions conn profile-id id))]
+                  (bfc/get-file-permissions conn profile-id id))]
     (check-read-permissions! perms)
 
     (let [team (teams/get-team conn
@@ -309,7 +240,7 @@
    ::sm/result schema:file-fragment}
   [cfg {:keys [::rpc/profile-id file-id fragment-id share-id]}]
   (db/run! cfg (fn [cfg]
-                 (let [perms (get-permissions cfg profile-id file-id share-id)]
+                 (let [perms (bfc/get-file-permissions cfg profile-id file-id share-id)]
                    (check-read-permissions! perms)
                    (-> (get-file-fragment cfg file-id fragment-id)
                        (rph/with-http-cache long-cache-duration))))))
@@ -353,9 +284,8 @@
    ::sm/params schema:get-project-files
    ::sm/result schema:files}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id]}]
-  (dm/with-open [conn (db/open pool)]
+  (projects/check-read-permissions! pool profile-id project-id)
-    (projects/check-read-permissions! conn profile-id project-id)
+  (get-project-files pool project-id))
-    (get-project-files conn project-id)))
 
 ;; --- COMMAND QUERY: has-file-libraries
 
@@ -424,7 +354,6 @@
 
 ;; --- QUERY COMMAND: get-page
 
-
 (defn- prune-objects
   "Given the page data and the object-id returns the page data with all
   other not needed objects removed from the `:objects` data
@@ -456,8 +385,7 @@
               :code :params-validation
               :hint "page-id is required when object-id is provided"))
 
-  (let [perms (get-permissions conn profile-id file-id share-id)
+  (let [perms (bfc/get-file-permissions conn profile-id file-id share-id)
-
         file  (bfc/get-file cfg file-id :read-only? true)
 
         proj  (db/get conn :project {:id (:project-id file)})
@@ -688,11 +616,10 @@
   "Get libraries used by the specified file."
   {::doc/added "1.17"
    ::sm/params schema:get-file-libraries}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
+  [cfg {:keys [::rpc/profile-id file-id]}]
-  (dm/with-open [conn (db/open pool)]
+  (bfc/check-file-exists cfg file-id)
-    (check-read-permissions! conn profile-id file-id)
+  (check-read-permissions! cfg profile-id file-id)
-    (bfc/get-file-libraries conn file-id)))
+  (bfc/get-file-libraries cfg file-id))
-
 
 ;; --- COMMAND QUERY: Files that use this File library
 
@@ -765,6 +692,52 @@
     (teams/check-read-permissions! conn profile-id team-id)
     (get-team-recent-files conn team-id)))
 
+
+;; --- COMMAND QUERY: get-team-deleted-files
+
+(def sql:team-deleted-files
+  "WITH deleted_files AS (
+     SELECT f.id,
+            f.revn,
+            f.vern,
+            f.project_id,
+            f.created_at,
+            f.modified_at,
+            f.name,
+            f.deleted_at AS will_be_deleted_at,
+            ft.media_id AS thumbnail_id,
+            row_number() OVER w AS row_num,
+            p.team_id
+       FROM file AS f
+      INNER JOIN project AS p ON (p.id = f.project_id)
+       LEFT JOIN file_thumbnail AS ft on (ft.file_id = f.id
+                                          AND ft.revn = f.revn)
+      WHERE p.team_id = ?
+        AND (p.deleted_at > ?::timestamptz OR
+             f.deleted_at > ?::timestamptz)
+     WINDOW w AS (PARTITION BY f.project_id
+                      ORDER BY f.modified_at DESC)
+      ORDER BY f.modified_at DESC
+   )
+   SELECT * FROM deleted_files")
+
+(defn get-team-deleted-files
+  [conn team-id]
+  (let [now (ct/now)]
+    (db/exec! conn [sql:team-deleted-files team-id now now])))
+
+(def ^:private schema:get-team-deleted-files
+  [:map {:title "get-team-deleted-files"}
+   [:team-id ::sm/uuid]])
+
+(sv/defmethod ::get-team-deleted-files
+  {::doc/added "2.12"
+   ::sm/params schema:get-team-deleted-files}
+  [cfg {:keys [::rpc/profile-id team-id]}]
+  (db/run! cfg (fn [{:keys [::db/conn]}]
+                 (teams/check-read-permissions! conn profile-id team-id)
+                 (get-team-deleted-files conn team-id))))
+
 ;; --- COMMAND QUERY: get-file-info
 
 
@@ -840,7 +813,7 @@
       AND (f.deleted_at IS NULL OR f.deleted_at > now())
     ORDER BY f.created_at ASC;")
 
-(defn- absorb-library-by-file!
+(defn- absorb-library-by-file
   [cfg ldata file-id]
 
   (assert (db/connection-map? cfg)
@@ -864,7 +837,7 @@
                              :modified-at (ct/now)
                              :has-media-trimmed false}))))
 
-(defn- absorb-library
+(defn- absorb-library*
   "Find all files using a shared library, and absorb all library assets
   into the file local libraries"
   [cfg {:keys [id data] :as library}]
@@ -879,10 +852,10 @@
            :library-id (str id)
            :files (str/join "," (map str ids)))
 
-    (run! (partial absorb-library-by-file! cfg data) ids)
+    (run! (partial absorb-library-by-file cfg data) ids)
     library))
 
-(defn absorb-library!
+(defn absorb-library
   [{:keys [::db/conn] :as cfg} id]
   (let [file (-> (bfc/get-file cfg id
                                :realize? true
@@ -899,7 +872,7 @@
     (-> (cfeat/get-team-enabled-features cf/flags team)
         (cfeat/check-file-features! (:features file)))
 
-    (absorb-library cfg file)))
+    (absorb-library* cfg file)))
 
 (defn- set-file-shared
   [{:keys [::db/conn] :as cfg} {:keys [profile-id id] :as params}]
@@ -912,14 +885,14 @@
                ;; file, we need to perform more complex operation,
                ;; so in this case we retrieve the complete file and
                ;; perform all required validations.
-               (let [file (-> (absorb-library! cfg id)
+               (let [file (-> (absorb-library cfg id)
                               (assoc :is-shared false))]
                  (db/delete! conn :file-library-rel {:library-file-id id})
                  (db/update! conn :file
                              {:is-shared false
                               :modified-at (ct/now)}
                              {:id id})
-                 (select-keys file [:id :name :is-shared]))
+                 file)
 
                (and (false? (:is-shared file))
                     (true? (:is-shared params)))
@@ -966,6 +939,11 @@
                           {:id file-id}
                           {::db/return-keys [:id :name :is-shared :deleted-at
                                              :project-id :created-at :modified-at]})]
+
+    ;; Remove all possible relations for that file
+    (db/delete! conn :file-library-rel
+                {:library-file-id file-id})
+
     (wrk/submit! {::db/conn conn
                   ::wrk/task :delete-object
                   ::wrk/params {:object :file
@@ -1113,3 +1091,144 @@
   (check-edition-permissions! conn profile-id file-id)
   (->  (ignore-sync conn params)
        (update :features db/decode-pgarray #{})))
+
+;; --- MUTATION COMMAND: delete-files-immediatelly
+
+(def ^:private sql:get-delete-team-files-candidates
+  "SELECT f.id
+    FROM file AS f
+    JOIN project AS p ON (p.id = f.project_id)
+    JOIN team AS t ON (t.id = p.team_id)
+   WHERE t.deleted_at IS NULL
+     AND t.id = ?
+     AND f.id = ANY(?::uuid[])")
+
+(def ^:private schema:permanently-delete-team-files
+  [:map {:title "permanently-delete-team-files"}
+   [:team-id ::sm/uuid]
+   [:ids [::sm/set ::sm/uuid]]])
+
+(defn- permanently-delete-team-files
+  [{:keys [::db/conn]} {:keys [::rpc/request-at team-id ids]}]
+  (let [ids (into #{}
+                  d/xf:map-id
+                  (db/exec! conn [sql:get-delete-team-files-candidates team-id
+                                  (db/create-array conn "uuid" ids)]))]
+
+    (reduce (fn [acc id]
+              (events/tap :progress {:file-id id :index (inc (count acc)) :total (count ids)})
+              (db/update! conn :file
+                          {:deleted-at request-at}
+                          {:id id}
+                          {::db/return-keys false})
+              (wrk/submit! {::db/conn conn
+                            ::wrk/task :delete-object
+                            ::wrk/params {:object :file
+                                          :deleted-at request-at
+                                          :id id}})
+              (conj acc id))
+            #{}
+            ids)))
+
+(sv/defmethod ::permanently-delete-team-files
+  "Mark the specified files to be deleted immediatelly on the
+  specified team. The team-id on params will be used to filter and
+  check writable permissons on team."
+
+  {::doc/added "2.13"
+   ::sm/params schema:permanently-delete-team-files}
+
+  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id] :as params}]
+  (teams/check-edition-permissions! pool profile-id team-id)
+  (sse/response #(db/tx-run! cfg permanently-delete-team-files params)))
+
+;; --- MUTATION COMMAND: restore-files-immediatelly
+
+(def ^:private sql:resolve-editable-files
+  "SELECT f.id, f.project_id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+     JOIN team AS t ON (t.id = p.team_id)
+    WHERE t.deleted_at IS NULL
+      AND t.id = ?
+      AND f.id = ANY(?::uuid[])")
+
+(defn- restore-file
+  [conn file-id]
+  (db/update! conn :file
+              {:deleted-at nil
+               :has-media-trimmed false}
+              {:id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-media-object
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-change
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-data
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-thumbnail
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-tagged-object-thumbnail
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false}))
+
+(def ^:private sql:restore-projects
+  "UPDATE project SET deleted_at = null WHERE id = ANY(?::uuid[])")
+
+(defn- restore-projects
+  [conn project-ids]
+  (let [project-ids (db/create-array conn "uuid" project-ids)]
+    (->> (db/exec-one! conn [sql:restore-projects project-ids])
+         (db/get-update-count))))
+
+(defn- restore-deleted-team-files
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id ids]}]
+  (teams/check-edition-permissions! conn profile-id team-id)
+  (let [total-files
+        (count ids)
+
+        {:keys [files projects]}
+        (reduce (fn [result {:keys [id project-id]}]
+                  (let [index (-> result :files count)]
+                    (events/tap :progress {:file-id id :index (inc index) :total total-files})
+                    (restore-file conn id)
+
+                    (-> result
+                        (update :files conj id)
+                        (update :projects conj project-id))))
+
+                {:files #{} :projectes #{}}
+                (db/plan conn [sql:resolve-editable-files team-id
+                               (db/create-array conn "uuid" ids)]))]
+
+    (restore-projects conn projects)
+
+    files))
+
+(def ^:private schema:restore-deleted-team-files
+  [:map {:title "restore-deleted-team-files"}
+   [:team-id ::sm/uuid]
+   [:ids [::sm/set ::sm/uuid]]])
+
+(sv/defmethod ::restore-deleted-team-files
+  "Removes the deletion mark from the specified files (and respective
+  projects) on the specified team."
+  {::doc/added "2.13"
+   ::sse/stream? true
+   ::sm/params schema:restore-deleted-team-files}
+  [cfg params]
+  (sse/response #(db/tx-run! cfg restore-deleted-team-files params)))

backend/src/app/rpc/commands/files_thumbnails.clj

@@ -96,7 +96,7 @@
 ;; loading all pages into memory for find the frame set for thumbnail.
 
 (defn get-file-data-for-thumbnail
-  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
+  [{:keys [::db/conn] :as cfg} {:keys [data id] :as file} strip-frames-with-thumbnails]
   (letfn [;; function responsible on finding the frame marked to be
           ;; used as thumbnail; the returned frame always have
           ;; the :page-id set to the page that it belongs.
@@ -173,7 +173,7 @@
 
         ;; Assoc the available thumbnails and prune not visible shapes
         ;; for avoid transfer unnecessary data.
-        :always
+        strip-frames-with-thumbnails
         (update :objects assoc-thumbnails page-id thumbs)))))
 
 (def ^:private
@@ -186,7 +186,8 @@
   [:map {:title "PartialFile"}
    [:id ::sm/uuid]
    [:revn {:min 0} ::sm/int]
-   [:page [:map-of :keyword ::sm/any]]])
+   [:page [:map-of :keyword ::sm/any]]
+   [:strip-frames-with-thumbnails {:optional true} ::sm/boolean]])
 
 (sv/defmethod ::get-file-data-for-thumbnail
   "Retrieves the data for generate the thumbnail of the file. Used
@@ -195,24 +196,26 @@
    ::doc/module :files
    ::sm/params schema:get-file-data-for-thumbnail
    ::sm/result schema:partial-file}
-  [cfg {:keys [::rpc/profile-id file-id] :as params}]
+  [cfg {:keys [::rpc/profile-id file-id strip-frames-with-thumbnails] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
                  (files/check-read-permissions! conn profile-id file-id)
-
                  (let [team (teams/get-team conn
                                             :profile-id profile-id
                                             :file-id file-id)
-
                        file (bfc/get-file cfg file-id
+                                          :include-deleted? true
                                           :realize? true
-                                          :read-only? true)]
+                                          :read-only? true)
+                       strip-frames-with-thumbnails
+                       (or (nil? strip-frames-with-thumbnails) ;; if not present, default to true
+                           (true? strip-frames-with-thumbnails))]
 
                    (-> (cfeat/get-team-enabled-features cf/flags team)
                        (cfeat/check-file-features! (:features file)))
 
                    {:file-id file-id
                     :revn (:revn file)
-                    :page (get-file-data-for-thumbnail cfg file)}))))
+                    :page (get-file-data-for-thumbnail cfg file strip-frames-with-thumbnails)}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MUTATION COMMANDS
@@ -328,12 +331,16 @@
 
 ;; --- MUTATION COMMAND: create-file-thumbnail
 
-(defn- create-file-thumbnail!
+(defn- create-file-thumbnail
-  [{:keys [::db/conn ::sto/storage]} {:keys [file-id revn props media] :as params}]
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id revn props media] :as params}]
   (media/validate-media-type! media)
   (media/validate-media-size! media)
 
-  (let [props (db/tjson (or props {}))
+  (let [file  (bfc/get-file cfg file-id
+                            :include-deleted? true
+                            :load-data? false)
+
+        props (db/tjson (or props {}))
         path  (:path media)
         mtype (:mtype media)
         hash  (sto/calculate-hash path)
@@ -362,7 +369,7 @@
 
         (db/update! conn :file-thumbnail
                     {:media-id (:id media)
-                     :deleted-at nil
+                     :deleted-at (:deleted-at file)
                      :updated-at tnow
                      :props props}
                     {:file-id file-id
@@ -373,6 +380,7 @@
                    :revn revn
                    :created-at tnow
                    :updated-at tnow
+                   :deleted-at (:deleted-at file)
                    :props props
                    :media-id (:id media)}))
 
@@ -397,6 +405,8 @@
    ::rtry/when rtry/conflict-exception?
    ::sm/params schema:create-file-thumbnail}
 
+  ;; FIXME: do not run the thumbnail upload inside a transaction
+
   [cfg {:keys [::rpc/profile-id file-id] :as params}]
   (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
                     ;; TODO For now we check read permissions instead of write,
@@ -404,6 +414,6 @@
                     ;; review this approach on the future.
                     (files/check-read-permissions! conn profile-id file-id)
                     (when-not (db/read-only? conn)
-                      (let [media (create-file-thumbnail! cfg params)]
+                      (let [media (create-file-thumbnail cfg params)]
                         {:uri (files/resolve-public-uri (:id media))
                          :id (:id media)})))))

backend/src/app/rpc/commands/fonts.clj

@@ -6,6 +6,7 @@
 
 (ns app.rpc.commands.fonts
   (:require
+   [app.binfile.common :as bfc]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
@@ -66,7 +67,7 @@
       (uuid? file-id)
       (let [file    (db/get-by-id conn :file file-id {:columns [:id :project-id]})
             project (db/get-by-id conn :project (:project-id file) {:columns [:id :team-id]})
-            perms   (files/get-permissions conn profile-id file-id share-id)]
+            perms   (bfc/get-file-permissions conn profile-id file-id share-id)]
         (files/check-read-permissions! perms)
         (db/query conn :team-font-variant
                   {:team-id (:team-id project)

backend/src/app/rpc/commands/ldap.clj

@@ -66,12 +66,12 @@
                             :member-email (:email profile))
               token  (tokens/generate cfg claims)]
           (-> {:invitation-token token}
-              (rph/with-transform (session/create-fn cfg (:id profile)))
+              (rph/with-transform (session/create-fn cfg profile))
               (rph/with-meta {::audit/props (:props profile)
                               ::audit/profile-id (:id profile)})))
 
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile))
             (rph/with-meta {::audit/props (:props profile)
                             ::audit/profile-id (:id profile)}))))))
 
@@ -83,6 +83,6 @@
                              (profile/clean-email)
                              (profile/get-profile-by-email conn))
                     (->> (assoc info :is-active true :is-demo false)
-                         (auth/create-profile! conn)
+                         (auth/create-profile cfg)
-                         (auth/create-profile-rels! conn)
+                         (auth/create-profile-rels conn)
                          (profile/strip-private-attrs))))))

backend/src/app/rpc/commands/media.clj

@@ -7,14 +7,10 @@
 (ns app.rpc.commands.media
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
-   [app.http.client :as http]
    [app.loggers.audit :as-alias audit]
    [app.media :as media]
    [app.rpc :as-alias rpc]
@@ -22,13 +18,7 @@
    [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
    [app.storage :as sto]
-   [app.storage.tmp :as tmp]
+   [app.util.services :as sv]))
-   [app.util.services :as sv]
-   [cuerdas.core :as str]
-   [datoteka.io :as io]))
-
-(def default-max-file-size
-  (* 1024 1024 10)) ; 10 MiB
 
 (def thumbnail-options
   {:width 100
@@ -197,56 +187,12 @@
 
     mobj))
 
-(defn download-image
-  [{:keys [::http/client]} uri]
-  (letfn [(parse-and-validate [{:keys [headers] :as response}]
-            (let [size     (some-> (get headers "content-length") d/parse-integer)
-                  mtype    (get headers "content-type")
-                  format   (cm/mtype->format mtype)
-                  max-size (cf/get :media-max-file-size default-max-file-size)]
-
-              (when-not size
-                (ex/raise :type :validation
-                          :code :unknown-size
-                          :hint "seems like the url points to resource with unknown size"))
-
-              (when (> size max-size)
-                (ex/raise :type :validation
-                          :code :file-too-large
-                          :hint (str/ffmt "the file size % is greater than the maximum %"
-                                          size
-                                          default-max-file-size)))
-
-              (when (nil? format)
-                (ex/raise :type :validation
-                          :code :media-type-not-allowed
-                          :hint "seems like the url points to an invalid media object"))
-
-              {:size size :mtype mtype :format format}))]
-
-    (let [{:keys [body] :as response} (http/req! client
-                                                 {:method :get :uri uri}
-                                                 {:response-type :input-stream :sync? true})
-          {:keys [size mtype]} (parse-and-validate response)
-          path    (tmp/tempfile :prefix "penpot.media.download.")
-          written (io/write* path body :size size)]
-
-      (when (not= written size)
-        (ex/raise :type :internal
-                  :code :mismatch-write-size
-                  :hint "unexpected state: unable to write to file"))
-
-      {:filename "tempfile"
-       :size size
-       :path path
-       :mtype mtype})))
-
 (defn- create-file-media-object-from-url
   [cfg {:keys [url name] :as params}]
-  (let [content (download-image cfg url)
+  (let [content (media/download-image cfg url)
         params  (-> params
                     (assoc :content content)
-                    (assoc :name (or name (:filename content))))]
+                    (assoc :name (d/nilv name "unknown")))]
 
     ;; NOTE: we use the climit here in a dynamic invocation because we
     ;; don't want saturate the process-image limit with IO (download

backend/src/app/rpc/commands/profile.clj

@@ -107,7 +107,9 @@
 (defn get-profile
   "Get profile by id. Throws not-found exception if no profile found."
   [conn id & {:as opts}]
-  (-> (db/get-by-id conn :profile id opts)
+  ;; NOTE: We need to set ::db/remove-deleted to false because demo profiles
+  ;; are created with a set deleted-at value
+  (-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false))
       (decode-row)))
 
 ;; --- MUTATION: Update Profile (own)
@@ -152,7 +154,6 @@
 
 (declare validate-password!)
 (declare update-profile-password!)
-(declare invalidate-profile-session!)
 
 (def ^:private
   schema:update-profile-password
@@ -167,8 +168,7 @@
    ::climit/id :auth/global
    ::db/transaction true}
   [cfg {:keys [::rpc/profile-id password] :as params}]
-  (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
+  (let [profile (validate-password! cfg (assoc params :profile-id profile-id))]
-        session-id (::session/id params)]
 
     (when (= (:email profile) (str/lower (:password params)))
       (ex/raise :type :validation
@@ -176,14 +176,12 @@
                 :hint "you can't use your email as password"))
 
     (update-profile-password! cfg (assoc profile :password password))
-    (invalidate-profile-session! cfg profile-id session-id)
-    nil))
 
-(defn- invalidate-profile-session!
+    (->> (rph/get-request params)
-  "Removes all sessions except the current one."
+         (session/get-session)
-  [{:keys [::db/conn]} profile-id session-id]
+         (session/invalidate-others cfg))
-  (let [sql "delete from http_session where profile_id = ? and id != ?"]
+
-    (:next.jdbc/update-count (db/exec-one! conn [sql profile-id session-id]))))
+    nil))
 
 (defn- validate-password!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
@@ -282,9 +280,9 @@
                          :file-path (str (:path file))
                          :file-mtype (:mtype file)}}))))
 
-(defn- generate-thumbnail!
+(defn- generate-thumbnail
-  [_ file]
+  [_ input]
-  (let [input   (media/run {:cmd :info :input file})
+  (let [input   (media/run {:cmd :info :input input})
         thumb   (media/run {:cmd :profile-thumbnail
                             :format :jpeg
                             :quality 85
@@ -305,7 +303,7 @@
                    (assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
                                        [:process-image/global]])
                    (assoc ::climit/label "upload-photo")
-                   (climit/invoke! generate-thumbnail! file))]
+                   (climit/invoke! generate-thumbnail file))]
     (sto/put-object! storage params)))
 
 ;; --- MUTATION: Request Email Change
@@ -473,13 +471,16 @@
           p.fullname AS name,
           p.email AS email
      FROM team_profile_rel AS tpr1
+     JOIN team as t
+       ON tpr1.team_id = t.id
      JOIN team_profile_rel AS tpr2
        ON (tpr1.team_id = tpr2.team_id)
      JOIN profile AS p
        ON (tpr2.profile_id = p.id)
     WHERE tpr1.profile_id = ?
       AND tpr1.is_owner IS true
-      AND tpr2.can_edit IS true")
+      AND tpr2.can_edit IS true
+      AND t.deleted_at IS NULL")
 
 (sv/defmethod ::get-subscription-usage
   {::doc/added "2.9"}

backend/src/app/rpc/commands/projects.clj

@@ -70,7 +70,27 @@
 
 ;; --- QUERY: Get projects
 
-(declare get-projects)
+(def ^:private sql:projects
+  "SELECT p.*,
+          coalesce(tpp.is_pinned, false) as is_pinned,
+          (SELECT count(*) FROM file AS f
+            WHERE f.project_id = p.id
+              AND f.deleted_at is null) AS count,
+          (SELECT count(*) FROM file AS f
+            WHERE f.project_id = p.id) AS total_count
+     FROM project AS p
+    INNER JOIN team AS t ON (t.id = p.team_id)
+     LEFT JOIN team_project_profile_rel AS tpp
+            ON (tpp.project_id = p.id AND
+                tpp.team_id = p.team_id AND
+                tpp.profile_id = ?)
+    WHERE p.team_id = ?
+      AND t.deleted_at is null
+    ORDER BY p.modified_at DESC")
+
+(defn get-projects
+  [conn profile-id team-id]
+  (db/exec! conn [sql:projects profile-id team-id]))
 
 (def ^:private schema:get-projects
   [:map {:title "get-projects"}
@@ -78,32 +98,11 @@
 
 (sv/defmethod ::get-projects
   {::doc/added "1.18"
+   ::doc/changes [["2.12" "This endpoint now return deleted but recoverable projects"]]
    ::sm/params schema:get-projects}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id team-id]}]
+  [cfg {:keys [::rpc/profile-id team-id]}]
-  (dm/with-open [conn (db/open pool)]
+  (teams/check-read-permissions! cfg profile-id team-id)
-    (teams/check-read-permissions! conn profile-id team-id)
+  (get-projects cfg profile-id team-id))
-    (get-projects conn profile-id team-id)))
-
-(def sql:projects
-  "select p.*,
-          coalesce(tpp.is_pinned, false) as is_pinned,
-          (select count(*) from file as f
-            where f.project_id = p.id
-              and deleted_at is null) as count
-     from project as p
-    inner join team as t on (t.id = p.team_id)
-     left join team_project_profile_rel as tpp
-            on (tpp.project_id = p.id and
-                tpp.team_id = p.team_id and
-                tpp.profile_id = ?)
-    where p.team_id = ?
-      and p.deleted_at is null
-      and t.deleted_at is null
-    order by p.modified_at desc")
-
-(defn get-projects
-  [conn profile-id team-id]
-  (db/exec! conn [sql:projects profile-id team-id]))
 
 ;; --- QUERY: Get all projects
 
@@ -170,12 +169,19 @@
 ;; --- MUTATION: Create Project
 
 (defn- create-project
-  [{:keys [::db/conn] :as cfg} {:keys [profile-id team-id] :as params}]
+  [{:keys [::db/conn] :as cfg} {:keys [::rpc/request-at profile-id team-id] :as params}]
-  (let [project (teams/create-project conn params)]
+  (assert (ct/inst? request-at) "expect request-at assigned")
+  (let [params    (-> params
+                      (assoc :created-at request-at)
+                      (assoc :modified-at request-at))
+        project   (teams/create-project conn params)
+        timestamp (::rpc/request-at params)]
     (teams/create-project-role conn profile-id (:id project) :owner)
     (db/insert! conn :team-project-profile-rel
                 {:project-id (:id project)
                  :profile-id profile-id
+                 :created-at timestamp
+                 :modified-at timestamp
                  :team-id team-id
                  :is-pinned false})
     (assoc project :is-pinned false)))

backend/src/app/rpc/commands/teams.clj

@@ -37,14 +37,14 @@
 ;; --- Helpers & Specs
 
 (def ^:private sql:team-permissions
-  "select tpr.is_owner,
+  "SELECT tpr.is_owner,
           tpr.is_admin,
           tpr.can_edit
-     from team_profile_rel as tpr
+     FROM team_profile_rel AS tpr
-     join team as t on (t.id = tpr.team_id)
+     JOIN team AS t ON (t.id = tpr.team_id)
-    where tpr.profile_id = ?
+    WHERE tpr.profile_id = ?
-      and tpr.team_id = ?
+      AND tpr.team_id = ?
-      and t.deleted_at is null")
+      AND t.deleted_at IS NULL")
 
 (defn get-permissions
   [conn profile-id team-id]

backend/src/app/rpc/commands/verify_token.clj

@@ -73,7 +73,7 @@
                   {:id (:id profile)}))
 
     (-> claims
-        (rph/with-transform (session/create-fn cfg profile-id))
+        (rph/with-transform (session/create-fn cfg profile))
         (rph/with-meta {::audit/name "verify-profile-email"
                         ::audit/props (audit/profile->props profile)
                         ::audit/profile-id (:id profile)}))))

backend/src/app/rpc/commands/viewer.clj

@@ -13,7 +13,6 @@
    [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
-   [app.rpc.commands.files :as files]
    [app.rpc.commands.teams :as teams]
    [app.rpc.cond :as-alias cond]
    [app.rpc.doc :as-alias doc]
@@ -121,7 +120,7 @@
   [system {:keys [::rpc/profile-id file-id share-id] :as params}]
   (db/run! system
            (fn [{:keys [::db/conn] :as system}]
-             (let [perms  (files/get-permissions conn profile-id file-id share-id)
+             (let [perms  (bfc/get-file-permissions conn profile-id file-id share-id)
                    params (-> params
                               (assoc ::perms perms)
                               (assoc :profile-id profile-id))]

backend/src/app/rpc/cond.clj

@@ -39,9 +39,8 @@
 (defn- encode
   [s]
   (-> s
-      bh/blake2b-256
+      (bh/blake2b-256)
-      bc/bytes->b64u
+      (bc/bytes->b64-str true)))
-      bc/bytes->str))
 
 (defn- fmt-key
   [s]

backend/src/app/rpc/doc.clj

@@ -16,6 +16,7 @@
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.openapi :as oapi]
    [app.common.schema.registry :as sr]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
@@ -25,7 +26,6 @@
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [integrant.core :as ig]
    [pretty-spec.core :as ps]
    [yetti.response :as-alias yres]))
 
@@ -33,8 +33,8 @@
 ;; DOC (human readable)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- prepare-doc-context
+(defn- context
-  [methods]
+  [{:keys [methods entrypoint label openapi]}]
   (letfn [(fmt-spec [mdata]
             (when-let [spec (ex/ignoring (s/spec (::sv/spec mdata)))]
               (with-out-str
@@ -62,8 +62,10 @@
              :added (::added mdata)
              :changes (some->> (::changes mdata) (partition-all 2) (map vec))
              :spec (fmt-spec mdata)
-             :entrypoint (str (cf/get :public-uri) "/api/rpc/command/" (::sv/name mdata))
+             :entrypoint (-> entrypoint
-
+                             (u/ensure-path-slash)
+                             (u/join (::sv/name mdata))
+                             (str))
              :params-schema-js   (fmt-schema :js mdata ::sm/params)
              :result-schema-js   (fmt-schema :js mdata ::sm/result)
              :webhook-schema-js  (fmt-schema :js mdata ::sm/webhook)
@@ -72,6 +74,9 @@
              :webhook-schema-clj (fmt-schema :clj mdata ::sm/webhook)})]
 
     {:version (:main cf/version)
+     :label label
+     :entrypoint (str entrypoint)
+     :openapi (str openapi)
      :methods
      (->> methods
           (map val)
@@ -80,17 +85,19 @@
           (map get-context)
           (sort-by (juxt :module :name)))}))
 
-(defn- doc-handler
+(defn- handler
-  [context]
+  [& {:keys [template] :as options}]
   (if (contains? cf/flags :backend-api-doc)
-    (fn [request]
+    (let [context  (delay (context options))
-      (let [params  (:query-params request)
+          template (or template "app/templates/api-doc.tmpl")]
-            pstyle  (:type params "js")
+      (fn [request]
-            context (assoc @context :param-style pstyle)]
+        (let [params  (:query-params request)
+              pstyle  (:type params "js")
+              context (assoc @context :param-style pstyle)]
 
-        {::yres/status 200
+          {::yres/status 200
-         ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
+           ::yres/body (-> (io/resource template)
-                         (tmpl/render context))}))
+                           (tmpl/render context))})))
     (fn [_]
       {::yres/status 404})))
 
@@ -98,8 +105,8 @@
 ;; OPENAPI / SWAGGER (v3.1)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn prepare-openapi-context
+(defn- openapi-context
-  [methods]
+  [{:keys [methods entrypoint description]}]
   (let [definitions (atom {})
         options {:registry sr/default-registry
                  ::oapi/definitions-path "#/components/schemas/"
@@ -112,7 +119,9 @@
         (fn [tsx schema]
           (let [schema  (sm/schema schema)
                 example (sm/generate schema)
-                example (sm/encode schema example output-transformer)]
+                example (sm/encode schema example output-transformer)
+                example (json/encode example :key-fn json/write-camel-key)]
+
             {:default
              {:description "A default response"
               :content
@@ -123,7 +132,9 @@
         gen-params-doc
         (fn [tsx schema]
           (let [example (sm/generate schema)
-                example (sm/encode schema example output-transformer)]
+                example (sm/encode schema example output-transformer)
+                example (json/encode example :key-fn json/write-camel-key)]
+
             {:required true
              :content
              {"application/json"
@@ -158,34 +169,35 @@
                (map gen-method-doc)
                (sort-by (juxt :module :name))
                (map (fn [doc]
-                      [(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
+                      [(:name doc) (:repr doc)]))
                (into {})))]
 
     {:openapi "3.0.0"
      :info {:version (:main cf/version)}
-     :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
+     :servers [{:url (str entrypoint)
-                ;; :description "penpot backend"
+                :description (or description "")}]
-                }]
      :paths paths
      :components {:schemas @definitions}}))
 
-(defn openapi-json-handler
+(defn- openapi-json-handler
-  [context]
+  [& {:as options}]
   (if (contains? cf/flags :backend-openapi-doc)
-    (fn [_]
+    (let [context (delay (openapi-context options))]
-      {::yres/status 200
+      (fn [_]
-       ::yres/headers {"content-type" "application/json; charset=utf-8"}
+        {::yres/status 200
-       ::yres/body (json/encode @context)})
+         ::yres/headers {"content-type" "application/json; charset=utf-8"}
+         ::yres/body (json/encode @context)}))
     (fn [_]
       {::yres/status 404})))
 
-(defn openapi-handler
+(defn- openapi-handler
-  []
+  [& {:keys [uri label]}]
   (if (contains? cf/flags :backend-openapi-doc)
     (fn [_]
       (let [swagger-js (slurp (io/resource "app/assets/swagger-ui-4.18.3.js"))
             swagger-cs (slurp (io/resource "app/assets/swagger-ui-4.18.3.css"))
-            context    {:public-uri (cf/get :public-uri)
+            context    {:uri (str uri)
+                        :label label
                         :swagger-js swagger-js
                         :swagger-css swagger-cs}]
         {::yres/status 200
@@ -196,27 +208,43 @@
       {::yres/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; MODULE INIT
+;; ROUTES HELPER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defmethod ig/assert-key ::routes
+(defn routes
-  [_ params]
+  [& {:keys [label base-uri description methods]}]
-  (assert (sm/valid? ::rpc/methods (::rpc/methods params)) "expected valid methods"))
+  (let [entrypoint
+        (-> base-uri
+            (u/ensure-path-slash)
+            (u/join "methods"))
 
-(defmethod ig/init-key ::routes
+        openapi
-  [_ {:keys [::rpc/methods] :as cfg}]
+        (-> base-uri
-  [(let [context (delay (prepare-doc-context methods))]
+            (u/ensure-path-slash)
-     [["/_doc"
+            (u/join "doc/openapi"))
-       {:handler (doc-handler context)
-        :allowed-methods #{:get}}]
-      ["/doc"
-       {:handler (doc-handler context)
-        :allowed-methods #{:get}}]])
 
-   (let [context (delay (prepare-openapi-context methods))]
+        template
-     [["/openapi"
+        (case label
-       {:handler (openapi-handler)
+          "management" "app/templates/management-api-doc.tmpl"
-        :allowed-methods #{:get}}]
+          "main"       "app/templates/main-api-doc.tmpl")]
-      ["/openapi.json"
+
-       {:handler (openapi-json-handler context)
+    ["/doc"
-        :allowed-methods #{:get}}]])])
+     ["" {:handler (handler :methods methods
+                            :label label
+                            :entrypoint entrypoint
+                            :openapi openapi
+                            :template template)
+          :allowed-methods #{:get}}]
+
+     ["/openapi"
+      {:handler (openapi-handler
+                 :uri (u/join openapi "openapi.json")
+                 :label label)
+       :allowed-methods #{:get}}]
+
+     ["/openapi.json"
+      {:handler (openapi-json-handler {:entrypoint entrypoint
+                                       :description description
+                                       :methods methods})
+
+       :allowed-methods #{:get}}]]))

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [yetti.response :as-alias yres]))
+   [yetti.response :as yres]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -78,3 +78,21 @@
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
                  (update response ::yres/headers assoc "cache-control" val)))))
+
+(defn stream
+  "A convenience allias for yetti.response/stream-body"
+  [f]
+  (yres/stream-body f))
+
+(defn get-request
+  "Get http request from RPC params"
+  [params]
+  (assert (contains? params ::rpc/request-at) "rpc params required")
+  (-> (meta params)
+      (get ::http/request)))
+
+(defn get-auth-data
+  "Get http auth-data from RPC params"
+  [params]
+  (-> (get-request params)
+      (get ::http/auth-data)))

backend/src/app/rpc/management/exporter.clj

@@ -0,0 +1,49 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.management.exporter
+  (:require
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [app.common.uri :as u]
+   [app.config :as cf]
+   [app.media :refer [schema:upload]]
+   [app.rpc :as-alias rpc]
+   [app.rpc.doc :as doc]
+   [app.storage :as sto]
+   [app.util.services :as sv]))
+
+;; ---- RPC METHOD: UPLOAD-TEMPFILE
+
+(def ^:private
+  schema:upload-tempfile-params
+  [:map {:title "upload-templfile-params"}
+   [:content schema:upload]])
+
+(def ^:private
+  schema:upload-tempfile-result
+  [:map {:title "upload-templfile-result"}])
+
+(sv/defmethod ::upload-tempfile
+  {::doc/added "2.12"
+   ::sm/params schema:upload-tempfile-params
+   ::sm/result schema:upload-tempfile-result}
+  [cfg {:keys [::rpc/profile-id content]}]
+  (let [storage (sto/resolve cfg)
+        hash    (sto/calculate-hash (:path content))
+        data    (-> (sto/content (:path content))
+                    (sto/wrap-with-hash hash))
+        content {::sto/content data
+                 ::sto/deduplicate? true
+                 ::sto/touched-at (ct/in-future {:minutes 10})
+                 :profile-id profile-id
+                 :content-type (:mtype content)
+                 :bucket "tempfile"}
+        object (sto/put-object! storage content)]
+    {:id (:id object)
+     :uri (-> (cf/get :public-uri)
+              (u/join "/assets/by-id/")
+              (u/join (str (:id object))))}))

backend/src/app/rpc/management/subscription.clj

@@ -0,0 +1,183 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.management.subscription
+  (:require
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.schema.generators :as sg]
+   [app.common.time :as ct]
+   [app.db :as db]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.profile :as profile]
+   [app.rpc.doc :as doc]
+   [app.util.services :as sv]))
+
+;; ---- RPC METHOD: AUTHENTICATE
+
+(def ^:private
+  schema:authenticate-params
+  [:map {:title "authenticate-params"}])
+
+(def ^:private
+  schema:authenticate-result
+  [:map {:title "authenticate-result"}
+   [:profile-id ::sm/uuid]])
+
+(sv/defmethod ::auth
+  {::doc/added "2.12"
+   ::sm/params schema:authenticate-params
+   ::sm/result schema:authenticate-result}
+  [_ {:keys [::rpc/profile-id]}]
+  {:profile-id profile-id})
+
+;; ---- RPC METHOD: GET-CUSTOMER
+
+;; FIXME: move to app.common.time
+(def ^:private schema:timestamp
+  (sm/type-schema
+   {:type ::timestamp
+    :pred ct/inst?
+    :type-properties
+    {:title "inst"
+     :description "The same as :app.common.time/inst but encodes to epoch"
+     :error/message "should be an instant"
+     :gen/gen (->> (sg/small-int)
+                   (sg/fmap (fn [v] (ct/inst v))))
+     :decode/string #(some-> % ct/inst)
+     :encode/string #(some-> % inst-ms)
+     :decode/json #(some-> % ct/inst)
+     :encode/json #(some-> % inst-ms)}}))
+
+(def ^:private schema:subscription
+  [:map {:title "Subscription"}
+   [:id ::sm/text]
+   [:customer-id ::sm/text]
+   [:type [:enum
+           "unlimited"
+           "professional"
+           "enterprise"]]
+   [:status [:enum
+             "active"
+             "canceled"
+             "incomplete"
+             "incomplete_expired"
+             "past_due"
+             "paused"
+             "trialing"
+             "unpaid"]]
+
+   [:billing-period [:enum
+                     "month"
+                     "day"
+                     "week"
+                     "year"]]
+   [:quantity :int]
+   [:description [:maybe ::sm/text]]
+   [:created-at schema:timestamp]
+   [:start-date [:maybe schema:timestamp]]
+   [:ended-at [:maybe schema:timestamp]]
+   [:trial-end [:maybe schema:timestamp]]
+   [:trial-start [:maybe schema:timestamp]]
+   [:cancel-at [:maybe schema:timestamp]]
+   [:canceled-at [:maybe schema:timestamp]]
+   [:current-period-end [:maybe schema:timestamp]]
+   [:current-period-start [:maybe schema:timestamp]]
+   [:cancel-at-period-end :boolean]
+
+   [:cancellation-details
+    [:map {:title "CancellationDetails"}
+     [:comment [:maybe ::sm/text]]
+     [:reason [:maybe ::sm/text]]
+     [:feedback [:maybe
+                 [:enum
+                  "customer_service"
+                  "low_quality"
+                  "missing_feature"
+                  "other"
+                  "switched_service"
+                  "too_complex"
+                  "too_expensive"
+                  "unused"]]]]]])
+
+(def ^:private sql:get-customer-slots
+  "WITH teams AS (
+    SELECT tpr.team_id AS id,
+           tpr.profile_id AS profile_id
+      FROM team_profile_rel AS tpr
+     WHERE tpr.is_owner IS true
+       AND tpr.profile_id = ?
+  ), teams_with_slots AS (
+    SELECT tpr.team_id AS id,
+           count(*) AS total
+      FROM team_profile_rel AS tpr
+     WHERE tpr.team_id IN (SELECT id FROM teams)
+       AND tpr.can_edit IS true
+     GROUP BY 1
+     ORDER BY 2
+  )
+  SELECT max(total) AS total FROM teams_with_slots;")
+
+(defn- get-customer-slots
+  [cfg profile-id]
+  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
+    (:total result)))
+
+(def ^:private schema:get-customer-params
+  [:map])
+
+(def ^:private schema:get-customer-result
+  [:map
+   [:id ::sm/uuid]
+   [:name :string]
+   [:num-editors ::sm/int]
+   [:subscription {:optional true} schema:subscription]])
+
+(sv/defmethod ::get-customer
+  {::doc/added "2.12"
+   ::sm/params schema:get-customer-params
+   ::sm/result schema:get-customer-result}
+  [cfg {:keys [::rpc/profile-id]}]
+  (let [profile (profile/get-profile cfg profile-id)]
+    {:id (get profile :id)
+     :name (get profile :fullname)
+     :email (get profile :email)
+     :num-editors (get-customer-slots cfg profile-id)
+     :subscription (-> profile :props :subscription)}))
+
+
+;; ---- RPC METHOD: GET-CUSTOMER
+
+(def ^:private schema:update-customer-params
+  [:map
+   [:subscription [:maybe schema:subscription]]])
+
+(def ^:private schema:update-customer-result
+  [:map])
+
+(sv/defmethod ::update-customer
+  {::doc/added "2.12"
+   ::sm/params schema:update-customer-params
+   ::sm/result schema:update-customer-result}
+  [cfg {:keys [::rpc/profile-id subscription]}]
+  (let [{:keys [props] :as profile}
+        (profile/get-profile cfg profile-id ::db/for-update true)
+
+        props
+        (assoc props :subscription subscription)]
+
+    (l/dbg :hint "update customer"
+           :profile-id (str profile-id)
+           :subscription-type (get subscription :type)
+           :subscription-status (get subscription :status)
+           :subscription-quantity (get subscription :quantity))
+
+    (db/update! cfg :profile
+                {:props (db/tjson props)}
+                {:id profile-id}
+                {::db/return-keys false})
+
+    nil))

backend/src/app/rpc/quotes.clj

@@ -102,8 +102,7 @@
                     ::wrk/label "quotes-notification"
                     ::wrk/params {:to (vec admins)
                                   :subject subject
-                                  :body [{:type "text/plain"
+                                  :body content}}))))
-                                          :content content}]}}))))
 
 (defn- generic-check!
   [{:keys [::db/conn ::incr ::quote-sql ::count-sql ::default ::target] :or {incr 1} :as params}]

backend/src/app/rpc/rlimit.clj

@@ -104,28 +104,29 @@
 (def ^:private schema:limit
   [:and
    [:map
-    [::name :any]
+    [::name :keyword]
     [::strategy schema:strategy]
     [::key :string]
-    [::opts :string]]
+    [::opts :string]
-   [:or
+    [::capacity {:optional true} ::sm/int]
-    [:map
+    [::rate {:optional true} ::sm/int]
-     [::capacity ::sm/int]
+    [::interval {:optional true} ::ct/duration]
-     [::rate ::sm/int]
+    [::params {:optional true} [::sm/vec :any]]
-     [::internal ::ct/duration]
+    [::permits {:optional true} ::sm/int]
-     [::params [::sm/vec :any]]]
+    [::unit {:optional true} [:enum :days :hours :minutes :seconds :weeks]]]
-    [:map
+   [:fn (fn [attrs]
-     [::nreq ::sm/int]
+          (let [contains-fn (partial contains? attrs)]
-     [::unit [:enum :days :hours :minutes :seconds :weeks]]]]])
+            (or (every? contains-fn [::capacity ::rate ::interval])
+                (every? contains-fn [::permits ::unit]))))]])
 
 (def ^:private schema:limits
   [:map-of :keyword [::sm/vec schema:limit]])
 
 (def ^:private valid-limit-tuple?
-  (sm/lazy-validator schema:limit-tuple))
+  (sm/validator schema:limit-tuple))
 
 (def ^:private valid-rlimit-instance?
-  (sm/lazy-validator ::rpc/rlimit))
+  (sm/validator ::rpc/rlimit))
 
 (defmethod parse-limit :window
   [[name strategy opts :as vlimit]]
@@ -134,16 +135,16 @@
   (merge
    {::name name
     ::strategy strategy}
-   (if-let [[_ nreq unit] (re-find window-opts-re opts)]
+   (if-let [[_ permits unit] (re-find window-opts-re opts)]
-     (let [nreq (parse-long nreq)]
+     (let [permits (parse-long permits)]
-       {::nreq nreq
+       {::permits permits
         ::unit (case unit
                  "d" :days
                  "h" :hours
                  "m" :minutes
                  "s" :seconds
                  "w" :weeks)
-        ::key  (str "ratelimit.window." (d/name name))
+        ::key  (str "penpot.rlimit." (cf/get :tenant) ".window." (d/name name))
         ::opts opts})
      (ex/raise :type :validation
                :code :invalid-window-limit-opts
@@ -164,15 +165,15 @@
        ::interval interval
        ::opts     opts
        ::params   [(->seconds interval) rate capacity]
-       ::key      (str "ratelimit.bucket." (d/name name))})
+       ::key      (str "penpot.rlimit." (cf/get :tenant) ".bucket." (d/name name))})
     (ex/raise :type :validation
               :code :invalid-bucket-limit-opts
               :hint (str/ffmt "looks like '%' does not have a valid format" opts))))
 
 (defmethod process-limit :bucket
-  [rconn user-id now {:keys [::key ::params ::service ::capacity ::interval ::rate] :as limit}]
+  [rconn profile-id now {:keys [::key ::params ::service ::capacity ::interval ::rate] :as limit}]
   (let [script    (-> bucket-rate-limit-script
-                      (assoc ::rscript/keys [(str key "." service "." user-id)])
+                      (assoc ::rscript/keys [(str key "." service "." profile-id)])
                       (assoc ::rscript/vals (conj params (->seconds now))))
         result    (rds/eval rconn script)
         allowed?  (boolean (nth result 0))
@@ -192,18 +193,18 @@
         (assoc ::lresult/remaining remaining))))
 
 (defmethod process-limit :window
-  [rconn user-id now {:keys [::nreq ::unit ::key ::service] :as limit}]
+  [rconn profile-id now {:keys [::permits ::unit ::key ::service] :as limit}]
   (let [ts        (ct/truncate now unit)
         ttl       (ct/diff now (ct/plus ts {unit 1}))
         script    (-> window-rate-limit-script
-                      (assoc ::rscript/keys [(str key "." service "." user-id "." (ct/format-inst ts))])
+                      (assoc ::rscript/keys [(str key "." service "." profile-id "." (ct/format-inst ts))])
-                      (assoc ::rscript/vals [nreq (->seconds ttl)]))
+                      (assoc ::rscript/vals [permits (->seconds ttl)]))
         result    (rds/eval rconn script)
         allowed?  (boolean (nth result 0))
         remaining (nth result 1)]
     (l/trace :hint "limit processed"
              :service service
-             :limit (name (::name limit))
+             :name (name (::name limit))
              :strategy (name (::strategy limit))
              :opts (::opts limit)
              :allowed allowed?
@@ -214,8 +215,8 @@
         (assoc ::lresult/reset (ct/plus ts {unit 1})))))
 
 (defn- process-limits
-  [rconn user-id limits now]
+  [rconn profile-id limits now]
-  (let [results   (into [] (map (partial process-limit rconn user-id now)) limits)
+  (let [results   (into [] (map (partial process-limit rconn profile-id now)) limits)
         remaining (->> results
                        (d/index-by ::name ::lresult/remaining)
                        (uri/map->query-string))
@@ -227,7 +228,7 @@
 
     (when rejected
       (l/warn :hint "rejected rate limit"
-              :user-id (str user-id)
+              :profile-id (str profile-id)
               :limit-service (-> rejected ::service name)
               :limit-name (-> rejected ::name name)
               :limit-strategy (-> rejected ::strategy name)))
@@ -371,12 +372,9 @@
 (defn- on-refresh-error
   [_ cause]
   (when-not (instance? java.util.concurrent.RejectedExecutionException cause)
-    (if-let [explain (-> cause ex-data ex/explain)]
+    (l/warn :hint "unexpected exception on loading config"
-      (l/warn ::l/raw (str "unable to refresh config, invalid format:\n" explain)
+            :cause cause
-              ::l/sync? true)
+            ::l/sync? true)))
-      (l/warn :hint "unexpected exception on loading config"
-              :cause cause
-              ::l/sync? true))))
 
 (defn- get-config-path
   []

backend/src/app/rpc/rlimit/bucket.lua

@@ -25,9 +25,9 @@ local allowed = filled >= requested
 local newTokens = filled
 if allowed then
   newTokens = filled - requested
+  redis.call("hset", tokensKey, "tokens", newTokens, "timestamp", timestamp)
 end
 
-redis.call("hset", tokensKey, "tokens", newTokens, "timestamp", timestamp)
 redis.call("expire", tokensKey, ttl)
 
 return { allowed, newTokens }

backend/src/app/setup.clj

@@ -22,8 +22,7 @@
 (defn- generate-random-key
   []
   (-> (bn/random-bytes 64)
-      (bc/bytes->b64u)
+      (bc/bytes->b64-str true)))
-      (bc/bytes->str)))
 
 (defn- get-all-props
   [conn]
@@ -85,12 +84,11 @@
                       (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
                                          "all sessions on each restart, it is highly recommended setting up the "
                                          "PENPOT_SECRET_KEY environment variable")))
-
                     (let [secret (or key (generate-random-key))]
                       (-> (get-all-props conn)
                           (assoc :secret-key secret)
                           (assoc :tokens-key (keys/derive secret :salt "tokens"))
+                          (assoc :management-key (keys/derive secret :salt "management"))
                           (update :instance-id handle-instance-id conn (db/read-only? pool)))))))
 
-;; FIXME
+(sm/register! ::props [:map-of :keyword ::sm/any])
-(sm/register! ::props :any)

backend/src/app/setup/clock.clj

@@ -14,7 +14,9 @@
    [integrant.core :as ig])
   (:import
    java.time.Clock
-   java.time.Duration))
+   java.time.Duration
+   java.time.Instant
+   java.time.ZoneId))
 
 (defonce current
   (atom {:clock (Clock/systemDefaultZone)
@@ -36,6 +38,12 @@
   [_ _]
   (remove-watch current ::common))
 
+(defn fixed
+  "Get fixed clock, mainly used in tests"
+  [instant]
+  (Clock/fixed ^Instant (ct/inst instant)
+               ^ZoneId (ZoneId/of "Z")))
+
 (defn set-offset!
   [duration]
   (swap! current assoc :offset (some-> duration ct/duration)))

backend/src/app/setup/keys.clj

@@ -8,13 +8,13 @@
   "Keys derivation service."
   (:refer-clojure :exclude [derive])
   (:require
-   [app.common.spec :as us]
    [buddy.core.kdf :as bk]))
 
 (defn derive
   "Derive a key from secret-key"
   [secret-key & {:keys [salt size] :or {size 32}}]
-  (us/assert! ::us/not-empty-string secret-key)
+  (assert (string? secret-key) "expect string")
+  (assert (seq secret-key) "expect string")
   (let [engine (bk/engine {:key secret-key
                            :salt salt
                            :alg :hkdf

backend/src/app/srepl/cli.clj

@@ -61,8 +61,8 @@
                              :is-active is-active
                              :password password
                              :props {}}]
-               (->> (cmd.auth/create-profile! conn params)
+               (->> (cmd.auth/create-profile system params)
-                    (cmd.auth/create-profile-rels! conn)))))))
+                    (cmd.auth/create-profile-rels conn)))))))
 
 (defmethod exec-command "update-profile"
   [{:keys [fullname email password is-active]}]

backend/src/app/srepl/main.clj

@@ -25,6 +25,7 @@
    [app.db.sql :as-alias sql]
    [app.features.fdata :as fdata]
    [app.features.file-snapshots :as fsnap]
+   [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as main]
    [app.msgbus :as mbus]
@@ -567,48 +568,12 @@
                                           :id file-id})))
     :deleted))
 
-(defn- restore-file*
-  [{:keys [::db/conn]} file-id]
-  (db/update! conn :file
-              {:deleted-at nil
-               :has-media-trimmed false}
-              {:id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-media-object
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-change
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-data
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  ;; Mark thumbnails to be deleted
-  (db/update! conn :file-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-tagged-object-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  :restored)
-
 (defn restore-file!
   "Mark a file and all related objects as not deleted"
   [file-id]
   (let [file-id (h/parse-uuid file-id)]
     (db/tx-run! main/system
-                (fn [system]
+                (fn [{:keys [::db/conn] :as system}]
                   (when-let [file (db/get* system :file
                                            {:id file-id}
                                            {::db/remove-deleted false
@@ -622,7 +587,9 @@
                                                      :cause "explicit call to restore-file!"}
                                     ::audit/tracked-at (ct/now)})
 
-                    (restore-file* system file-id))))))
+
+                    (#'files/restore-file conn file-id))
+                  :restored))))
 
 (defn delete-project!
   "Mark a project for deletion"
@@ -655,7 +622,7 @@
   (doseq [{:keys [id]} (db/query conn :file
                                  {:project-id project-id}
                                  {::sql/columns [:id]})]
-    (restore-file* cfg id))
+    (#'files/restore-file conn id))
 
   :restored)
 
@@ -877,10 +844,33 @@
                                                      :deleted-at deleted-at
                                                      :id id})))))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SSO
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn add-sso-config
+  [& {:keys [base-uri client-id client-secret domain]}]
+
+  (assert (and (string? base-uri) (str/starts-with? base-uri "http")) "expected a valid base-uri")
+  (assert (string? client-id) "expected a valid client-id")
+  (assert (string? client-secret) "expected a valid client-secret")
+  (assert (string? domain) "expected a valid domain")
+  (db/insert! main/system :sso-provider
+              {:id (uuid/next)
+               :type "oidc"
+               :client-id client-id
+               :client-secret client-secret
+               :domain domain
+               :base-uri base-uri}))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MISC
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn decode-session-token
+  [token]
+  (session/decode-token main/system token))
+
 (defn instrument-var
   [var]
   (alter-var-root var (fn [f]

backend/src/app/storage.clj

@@ -35,12 +35,16 @@
       :assets-s3 :s3
       nil)))
 
+(def default-bucket
+  "file-media-object")
+
 (def valid-buckets
   #{"file-media-object"
     "team-font-variant"
     "file-object-thumbnail"
     "file-thumbnail"
     "profile"
+    "tempfile"
     "file-data"
     "file-data-fragment"
     "file-change"})
@@ -163,9 +167,6 @@
      backend
      (:metadata result))))
 
-(def ^:private sql:retrieve-storage-object
-  "select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
-
 (defn row->storage-object [res]
   (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
     (impl/storage-object
@@ -177,9 +178,15 @@
      (keyword (:backend res))
      mdata)))
 
-(defn- retrieve-database-object
+(def ^:private sql:get-storage-object
+  "SELECT *
+     FROM storage_object
+    WHERE id = ?
+      AND (deleted_at IS NULL)")
+
+(defn- get-database-object
   [conn id]
-  (some-> (db/exec-one! conn [sql:retrieve-storage-object id])
+  (some-> (db/exec-one! conn [sql:get-storage-object id])
           (row->storage-object)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -202,7 +209,7 @@
 (defn get-object
   [{:keys [::db/connectable] :as storage}  id]
   (assert (valid-storage? storage))
-  (retrieve-database-object connectable id))
+  (get-database-object connectable id))
 
 (defn put-object!
   "Creates a new object with the provided content."

backend/src/app/storage/gc_deleted.clj

@@ -37,7 +37,6 @@
          (into #{} (map :id))
          (not-empty))))
 
-
 (def ^:private sql:delete-sobjects
   "DELETE FROM storage_object
     WHERE id = ANY(?::uuid[])")
@@ -77,47 +76,37 @@
   (d/group-by (comp keyword :backend) :id #{} items))
 
 (def ^:private sql:get-deleted-sobjects
-  "SELECT s.* FROM storage_object AS s
+  "SELECT s.*
+     FROM storage_object AS s
     WHERE s.deleted_at IS NOT NULL
-      AND s.deleted_at < now() - ?::interval
+      AND s.deleted_at <= ?
     ORDER BY s.deleted_at ASC")
 
 (defn- get-buckets
-  [conn min-age]
+  [conn]
-  (let [age (db/interval min-age)]
+  (let [now (ct/now)]
     (sequence
      (comp (partition-all 25)
            (mapcat group-by-backend))
-     (db/cursor conn [sql:get-deleted-sobjects age]))))
+     (db/cursor conn [sql:get-deleted-sobjects now]))))
-
 
 (defn- clean-deleted!
-  [{:keys [::db/conn ::min-age] :as cfg}]
+  [{:keys [::db/conn] :as cfg}]
   (reduce (fn [total [backend-id ids]]
             (let [deleted (delete-in-bulk! cfg backend-id ids)]
               (+ total (or deleted 0))))
           0
-          (get-buckets conn min-age)))
+          (get-buckets conn)))
 
 (defmethod ig/assert-key ::handler
   [_ params]
   (assert (sto/valid-storage? (::sto/storage params)) "expect valid storage")
   (assert (db/pool? (::db/pool params)) "expect valid storage"))
 
-(defmethod ig/expand-key ::handler
-  [k v]
-  {k (assoc v ::min-age (ct/duration {:hours 2}))})
-
 (defmethod ig/init-key ::handler
-  [_ {:keys [::min-age] :as cfg}]
+  [_ cfg]
-  (fn [{:keys [props] :as task}]
+  (fn [_]
-    (let [min-age (ct/duration (or (:min-age props) min-age))]
+    (db/tx-run! cfg (fn [cfg]
-      (db/tx-run! cfg (fn [cfg]
+                      (let [total (clean-deleted! cfg)]
-                        (let [cfg   (assoc cfg ::min-age min-age)
+                        (l/inf :hint "task finished" :total total)
-                              total (clean-deleted! cfg)]
+                        {:deleted total})))))
-
-                          (l/inf :hint "task finished"
-                                 :min-age (ct/format-duration min-age)
-                                 :total total)
-
-                          {:deleted total}))))))

backend/src/app/storage/gc_touched.clj

@@ -22,8 +22,10 @@
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
+   [app.common.time :as ct]
+   [app.config :as cf]
    [app.db :as db]
-   [app.storage :as-alias sto]
+   [app.storage :as sto]
    [app.storage.impl :as impl]
    [integrant.core :as ig]))
 
@@ -101,14 +103,15 @@
 
 (def ^:private sql:mark-delete-in-bulk
   "UPDATE storage_object
-      SET deleted_at = now(),
+      SET deleted_at = ?,
           touched_at = NULL
     WHERE id = ANY(?::uuid[])")
 
 (defn- mark-delete-in-bulk!
-  [conn ids]
+  [conn deletion-delay ids]
-  (let [ids (db/create-array conn "uuid" ids)]
+  (let [ids (db/create-array conn "uuid" ids)
-    (db/exec-one! conn [sql:mark-delete-in-bulk ids])))
+        now (ct/plus (ct/now) deletion-delay)]
+    (db/exec-one! conn [sql:mark-delete-in-bulk now ids])))
 
 ;; NOTE: A getter that retrieves the key which will be used for group
 ;; ids; previously we have no value, then we introduced the
@@ -127,7 +130,7 @@
   [{:keys [metadata]}]
   (or (some-> metadata :bucket)
       (some-> metadata :reference d/name)
-      "file-media-object"))
+      sto/default-bucket))
 
 (defn- process-objects!
   [conn has-refs? bucket objects]
@@ -137,18 +140,20 @@
     (if-let [{:keys [id] :as object} (first objects)]
       (if (has-refs? conn object)
         (do
-          (l/debug :id (str id)
+          (l/dbg :id (str id)
-                   :status "freeze"
+                 :status "freeze"
-                   :bucket bucket)
+                 :bucket bucket)
           (recur (conj to-freeze id) to-delete (rest objects)))
         (do
-          (l/debug :id (str id)
+          (l/dbg :id (str id)
-                   :status "delete"
+                 :status "delete"
-                   :bucket bucket)
+                 :bucket bucket)
           (recur to-freeze (conj to-delete id) (rest objects))))
-      (do
+      (let [deletion-delay (if (= bucket "tempfile")
+                             (ct/duration {:hours 2})
+                             (cf/get-deletion-delay))]
         (some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
-        (some->> (seq to-delete) (mark-delete-in-bulk! conn))
+        (some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay))
         [(count to-freeze) (count to-delete)]))))
 
 (defn- process-bucket!
@@ -160,6 +165,7 @@
     "file-thumbnail"        (process-objects! conn has-file-thumbnails-refs? bucket objects)
     "profile"               (process-objects! conn has-profile-refs? bucket objects)
     "file-data"             (process-objects! conn has-file-data-refs? bucket objects)
+    "tempfile"              (process-objects! conn (constantly false) bucket objects)
     (ex/raise :type :internal
               :code :unexpected-unknown-reference
               :hint (dm/fmt "unknown reference '%'" bucket))))
@@ -173,27 +179,27 @@
              [0 0]
              (d/group-by lookup-bucket identity #{} chunk)))
 
-(def ^:private
+(def ^:private sql:get-touched-storage-objects
-  sql:get-touched-storage-objects
   "SELECT so.*
      FROM storage_object AS so
     WHERE so.touched_at IS NOT NULL
+      AND so.touched_at <= ?
     ORDER BY touched_at ASC
       FOR UPDATE
      SKIP LOCKED
     LIMIT 10")
 
 (defn get-chunk
-  [conn]
+  [conn timestamp]
-  (->> (db/exec! conn [sql:get-touched-storage-objects])
+  (->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
        (map impl/decode-row)
        (not-empty)))
 
 (defn- process-touched!
-  [{:keys [::db/pool] :as cfg}]
+  [{:keys [::db/pool ::timestamp] :as cfg}]
   (loop [freezed 0
          deleted 0]
-    (if-let [chunk (get-chunk pool)]
+    (if-let [chunk (get-chunk pool timestamp)]
       (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
         (recur (long (+ freezed nfo))
                (long (+ deleted ndo))))
@@ -209,5 +215,6 @@
 
 (defmethod ig/init-key ::handler
   [_ cfg]
-  (fn [_] (process-touched! cfg)))
+  (fn [_]
+    (process-touched! (assoc cfg ::timestamp (ct/now)))))
 

backend/src/app/storage/tmp.clj

@@ -79,14 +79,17 @@
 ;; API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn tempfile
+(defn tempfile*
-  [& {:keys [suffix prefix min-age]
+  [& {:keys [suffix prefix]
       :or {prefix "penpot."
            suffix ".tmp"}}]
   (let [attrs (fs/make-permissions "rw-r--r--")
-        path  (fs/join default-tmp-dir (str prefix (uuid/next) suffix))
+        path  (fs/join default-tmp-dir (str prefix (uuid/next) suffix))]
-        path  (Files/createFile path attrs)]
+    (Files/createFile path attrs)))
-    (fs/delete-on-exit! path)
+
+(defn tempfile
+  [& {:keys [min-age] :as opts}]
+  (let [path (tempfile* opts)]
     (sp/offer! queue [path (some-> min-age ct/duration)])
     path))
 

backend/src/app/tasks/delete_object.clj

@@ -45,7 +45,8 @@
            :deleted-at (ct/format-inst deleted-at))
 
     (db/update! conn :file
-                {:deleted-at deleted-at}
+                {:deleted-at deleted-at
+                 :is-shared false}
                 {:id id}
                 {::db/return-keys false})
 
@@ -53,7 +54,7 @@
                (not *team-deletion*))
       ;; NOTE: we don't prevent file deletion on absorb operation failure
       (try
-        (db/tx-run! cfg files/absorb-library! id)
+        (db/tx-run! cfg files/absorb-library id)
         (catch Throwable cause
           (l/warn :hint "error on absorbing library"
                   :file-id id

backend/src/app/tasks/file_gc.clj

@@ -218,6 +218,9 @@
     (when (or (nil? revn) (= revn (:revn file)))
       file)))
 
+;; FIXME: we should skip files that does not match the revn on the
+;; props and add proper schema for this task props
+
 (defn- process-file!
   [cfg {:keys [file-id] :as props}]
   (if-let [file (get-file cfg props)]

backend/src/app/tasks/file_gc_scheduler.clj

@@ -8,6 +8,7 @@
   "A maintenance task that is responsible of properly scheduling the
   file-gc task for all files that matches the eligibility threshold."
   (:require
+   [app.common.logging :as l]
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
@@ -21,25 +22,24 @@
           f.modified_at
      FROM file AS f
     WHERE f.has_media_trimmed IS false
-      AND f.modified_at < now() - ?::interval
+      AND f.modified_at < ?
       AND f.deleted_at IS NULL
     ORDER BY f.modified_at DESC
       FOR UPDATE OF f
      SKIP LOCKED")
 
-(defn- get-candidates
-  [{:keys [::db/conn ::min-age] :as cfg}]
-  (let [min-age (db/interval min-age)]
-    (db/plan conn [sql:get-candidates min-age] {:fetch-size 10})))
-
 (defn- schedule!
-  [cfg]
+  [{:keys [::db/conn] :as cfg} threshold]
   (let [total (reduce (fn [total {:keys [id modified-at revn]}]
-                        (let [params {:file-id id :modified-at modified-at :revn revn}]
+                        (let [params {:file-id id :revn revn}]
+                          (l/trc :hint "schedule"
+                                 :file-id (str id)
+                                 :revn revn
+                                 :modified-at (ct/format-inst modified-at))
                           (wrk/submit! (assoc cfg ::wrk/params params))
                           (inc total)))
                       0
-                      (get-candidates cfg))]
+                      (db/plan conn [sql:get-candidates threshold] {:fetch-size 10}))]
     {:processed total}))
 
 (defmethod ig/assert-key ::handler
@@ -53,12 +53,12 @@
 (defmethod ig/init-key ::handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (let [min-age (ct/duration (or (:min-age props) (::min-age cfg)))]
+    (let [threshold (-> (ct/duration (or (:min-age props) (::min-age cfg)))
+                        (ct/in-past))]
       (-> cfg
           (assoc ::db/rollback (:rollback? props))
-          (assoc ::min-age min-age)
           (assoc ::wrk/task :file-gc)
           (assoc ::wrk/priority 10)
           (assoc ::wrk/mark-retries 0)
-          (assoc ::wrk/delay 1000)
+          (assoc ::wrk/delay 10000)
-          (db/tx-run! schedule!)))))
+          (db/tx-run! schedule! threshold)))))

backend/src/app/tasks/objects_gc.clj

@@ -18,15 +18,15 @@
 (def ^:private sql:get-profiles
   "SELECT id, photo_id FROM profile
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-profiles!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id]}]
                  (l/trc :obj "profile" :id (str id))
 
@@ -41,15 +41,15 @@
 (def ^:private sql:get-teams
   "SELECT deleted_at, id, photo_id FROM team
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-teams!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id deleted-at]}]
                  (l/trc :obj "team"
                         :id (str id)
@@ -68,15 +68,15 @@
   "SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
      FROM team_font_variant
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-fonts!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at] :as font}]
                  (l/trc :obj "font-variant"
                         :id (str id)
@@ -98,15 +98,15 @@
   "SELECT id, deleted_at, team_id
      FROM project
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-projects!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at]}]
                  (l/trc :obj "project"
                         :id (str id)
@@ -124,15 +124,15 @@
           f.project_id
      FROM file AS f
     WHERE f.deleted_at IS NOT NULL
-      AND f.deleted_at < now() + ?::interval
+      AND f.deleted_at <= ?
     ORDER BY f.deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-files!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id deleted-at project-id] :as file}]
                  (l/trc :obj "file"
                         :id (str id)
@@ -148,15 +148,15 @@
   "SELECT file_id, revn, media_id, deleted_at
      FROM file_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-thumbnails!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
                  (l/trc :obj "file-thumbnail"
                         :file-id (str file-id)
@@ -175,15 +175,15 @@
   "SELECT file_id, object_id, media_id, deleted_at
      FROM file_tagged_object_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-object-thumbnails!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
                  (l/trc :obj "file-object-thumbnail"
                         :file-id (str file-id)
@@ -203,15 +203,15 @@
   "SELECT id, file_id, media_id, thumbnail_id, deleted_at
      FROM file_media_object
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-media-objects!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
                  (l/trc :obj "file-media-object"
                         :id (str id)
@@ -231,16 +231,15 @@
   "SELECT file_id, id, type, deleted_at, metadata, backend
      FROM file_data
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-data!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-
+  (->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5})
-  (->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
 
                  (some->> metadata
@@ -266,15 +265,15 @@
   "SELECT id, file_id, deleted_at
      FROM file_change
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-changes!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
                  (l/trc :obj "file-change"
                         :id (str id)
@@ -322,9 +321,8 @@
 
 (defmethod ig/init-key ::handler
   [_ cfg]
-  (fn [{:keys [props] :as task}]
+  (fn [_]
-    (let [threshold (ct/duration (get props :deletion-threshold 0))
+    (let [cfg (assoc cfg ::timestamp (ct/now))]
-          cfg       (assoc cfg ::deletion-threshold (db/interval threshold))]
       (loop [procs (map deref deletion-proc-vars)
              total 0]
         (if-let [proc-fn (first procs)]

backend/src/app/tokens.clj

@@ -15,19 +15,25 @@
    [buddy.sign.jwe :as jwe]))
 
 (defn generate
-  [{:keys [::setup/props] :as cfg} claims]
+  ([cfg claims] (generate cfg claims nil))
-  (assert (contains? cfg ::setup/props))
+  ([{:keys [::setup/props] :as cfg} claims header]
+   (assert (contains? props :tokens-key) "expect props to have tokens-key")
 
-  (let [tokens-key
+   (let [tokens-key
-        (get props :tokens-key)
+         (get props :tokens-key)
 
-        payload
+         payload
-        (-> claims
+         (-> claims
-            (update :iat (fn [v] (or v (ct/now))))
+             (update :iat (fn [v] (or v (ct/now))))
-            (d/without-nils)
+             (d/without-nils)
-            (t/encode))]
+             (t/encode))]
 
-    (jwe/encrypt payload tokens-key {:alg :a256kw :enc :a256gcm})))
+     (jwe/encrypt payload tokens-key {:alg :a256kw :enc :a256gcm :header header}))))
+
+(defn decode-header
+  [token]
+  (ex/ignoring
+   (jwe/decode-header token)))
 
 (defn decode
   [{:keys [::setup/props] :as cfg} token]

backend/src/app/util/services.clj

@@ -27,7 +27,7 @@
       (throw (IllegalArgumentException. "Missing arguments on `defmethod` macro.")))
 
     (let [mdata (assoc mdata
-                       ::docstring (some-> docs str/<<-)
+                       ::docstring (some-> docs str/unindent)
                        ::spec sname
                        ::name (name sname))
 

backend/src/app/util/template.clj

@@ -7,10 +7,18 @@
 (ns app.util.template
   (:require
    [app.common.exceptions :as ex]
+   [cuerdas.core :as str]
+   [selmer.filters :as sf]
    [selmer.parser :as sp]))
 
 ;; (sp/cache-off!)
 
+(sf/add-filter! :abbreviate
+                (fn [s n]
+                  (let [n (parse-long n)]
+                    (str/abbreviate s n))))
+
+
 (defn render
   [path context]
   (try

backend/src/app/worker.clj

@@ -77,8 +77,8 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private sql:insert-new-task
-  "insert into task (id, name, props, queue, label, priority, max_retries, scheduled_at)
+  "insert into task (id, name, props, queue, label, priority, max_retries, created_at, modified_at, scheduled_at)
-   values (?, ?, ?, ?, ?, ?, ?, now() + ?)
+   values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    returning id")
 
 (def ^:private
@@ -88,7 +88,7 @@
       AND queue=?
       AND label=?
       AND status = 'new'
-      AND scheduled_at > now()")
+      AND scheduled_at > ?")
 
 (def ^:private schema:options
   [:map {:title "submit-options"}
@@ -111,17 +111,19 @@
 
   (check-options! options)
 
-  (let [duration  (ct/duration delay)
+  (let [delay        (ct/duration delay)
-        interval  (db/interval duration)
+        now          (ct/now)
-        props     (db/tjson params)
+        scheduled-at (-> (ct/plus now delay)
-        id        (uuid/next)
+                         (ct/truncate :millisecond))
-        tenant    (cf/get :tenant)
+        props        (db/tjson params)
-        task      (d/name task)
+        id           (uuid/next)
-        queue     (str/ffmt "%:%" tenant (d/name queue))
+        tenant       (cf/get :tenant)
-        conn      (db/get-connectable options)
+        task         (d/name task)
-        deleted   (when dedupe
+        queue        (str/ffmt "%:%" tenant (d/name queue))
-                    (-> (db/exec-one! conn [sql:remove-not-started-tasks task queue label])
+        conn         (db/get-connectable options)
-                        :next.jdbc/update-count))]
+        deleted      (when dedupe
+                       (-> (db/exec-one! conn [sql:remove-not-started-tasks task queue label now])
+                           (db/get-update-count)))]
 
     (l/trc :hint "submit task"
            :name task
@@ -129,11 +131,13 @@
            :queue queue
            :label label
            :dedupe (boolean dedupe)
-           :delay (ct/format-duration duration)
+           :delay (ct/format-duration delay)
            :replace (or deleted 0))
 
     (db/exec-one! conn [sql:insert-new-task id task props queue
-                        label priority max-retries interval])
+                        label priority max-retries
+                        now now scheduled-at])
+
     id))
 
 (defn invoke!

backend/src/app/worker/dispatcher.clj

@@ -137,33 +137,34 @@ RETURNING task.id, task.queue")
                 ::wait)))
 
           (run-batch []
-            (let [rconn (rds/connect cfg)]
+            (try
-              (try
+              (let [rconn (rds/connect cfg)]
-                (-> cfg
+                (try
-                    (assoc ::rds/conn rconn)
+                  (-> cfg
-                    (db/tx-run! run-batch'))
+                      (assoc ::rds/conn rconn)
+                      (db/tx-run! run-batch'))
+                  (finally
+                    (.close ^AutoCloseable rconn))))
 
-                (catch InterruptedException cause
+              (catch InterruptedException cause
-                  (throw cause))
+                (throw cause))
-                (catch Exception cause
-                  (cond
-                    (rds/exception? cause)
-                    (do
-                      (l/wrn :hint "redis exception (will retry in an instant)" :cause cause)
-                      (px/sleep timeout))
 
-                    (db/sql-exception? cause)
+              (catch Exception cause
-                    (do
+                (cond
-                      (l/wrn :hint "database exception (will retry in an instant)" :cause cause)
+                  (rds/exception? cause)
-                      (px/sleep timeout))
+                  (do
+                    (l/wrn :hint "redis exception (will retry in an instant)" :cause cause)
+                    (px/sleep timeout))
 
-                    :else
+                  (db/sql-exception? cause)
-                    (do
+                  (do
-                      (l/err :hint "unhandled exception (will retry in an instant)" :cause cause)
+                    (l/wrn :hint "database exception (will retry in an instant)" :cause cause)
-                      (px/sleep timeout))))
+                    (px/sleep timeout))
 
-                (finally
+                  :else
-                  (.close ^AutoCloseable rconn)))))
+                  (do
+                    (l/err :hint "unhandled exception (will retry in an instant)" :cause cause)
+                    (px/sleep timeout))))))
 
           (dispatcher []
             (l/inf :hint "started")
@@ -176,7 +177,7 @@ RETURNING task.id, task.queue")
               (catch InterruptedException _
                 (l/trc :hint "interrupted"))
               (catch Throwable cause
-                (l/err :hint " unexpected exception" :cause cause))
+                (l/err :hint "unexpected exception" :cause cause))
               (finally
                 (l/inf :hint "terminated"))))]
 

backend/src/app/worker/runner.clj

@@ -13,6 +13,7 @@
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.transit :as t]
+   [app.config :as cf]
    [app.db :as db]
    [app.metrics :as mtx]
    [app.redis :as rds]
@@ -60,7 +61,8 @@
 
 (defn get-error-context
   [_ item]
-  {:params item})
+  (-> (cf/logging-context)
+      (assoc :params item)))
 
 (defn- get-task
   [{:keys [::db/pool]} task-id]
@@ -131,6 +133,11 @@
   [{:keys [::id ::timeout] :as cfg} task-id scheduled-at]
   (loop [task (get-task cfg task-id)]
     (cond
+      (nil? task)
+      (l/wrn :hint "no task found on the database"
+             :runner-id id
+             :task-id task-id)
+
       (ex/exception? task)
       (if (or (db/connection-error? task)
               (db/serialization-error? task))
@@ -151,12 +158,9 @@
             (inst-ms (:scheduled-at task)))
       (l/wrn :hint "skiping task, rescheduled"
              :task-id task-id
-             :runner-id id)
-
-      (nil? task)
-      (l/wrn :hint "no task found on the database"
              :runner-id id
-             :task-id task-id)
+             :scheduled-at (ct/format-inst (:scheduled-at task))
+             :expected-scheduled-at (ct/format-inst scheduled-at))
 
       :else
       (let [result (run-task cfg task)]
@@ -177,7 +181,8 @@
                           {:error explain
                            :status "retry"
                            :modified-at now
-                           :scheduled-at (ct/plus now delay)
+                           :scheduled-at (-> (ct/plus now delay)
+                                             (ct/truncate :millisecond))
                            :retry-num nretry}
                           {:id (:id task)})
               nil))
@@ -213,6 +218,7 @@
                          :payload payload)))
               (catch Throwable cause
                 (l/err :hint "unable to decode payload"
+                       ::l/context (cf/logging-context)
                        :payload payload
                        :length (alength ^String/1 payload)
                        :cause cause))))
@@ -224,11 +230,11 @@
                "failed"    (handle-task-failure result)
                "completed" (handle-task-completion result)
                (throw (IllegalArgumentException.
-                       (str "invalid status received: " status))))))
+                       (str "invalid status received: '" status "'"))))))
 
           (run-task-loop [[task-id scheduled-at]]
             (loop [result (run-task! cfg task-id scheduled-at)]
-              (when-let [cause (process-result result)]
+              (when-let [cause (some-> result process-result)]
                 (if (or (db/connection-error? cause)
                         (db/serialization-error? cause))
                   (do
@@ -236,9 +242,9 @@
                            :cause cause)
                     (px/sleep timeout)
                     (recur result))
-                  (do
+                  (l/err :hint "unhandled exception on processing task result"
-                    (l/err :hint "unhandled exception on processing task result"
+                         ::l/context (cf/logging-context)
-                           :cause cause))))))]
+                         :cause cause)))))]
 
     (try
       (let [key         (str/ffmt "penpot.worker.queue:%" queue)
@@ -254,11 +260,14 @@
         (if (rds/timeout-exception? cause)
           (do
             (l/err :hint "redis pop operation timeout, consider increasing redis timeout (will retry in some instants)"
+                   ::l/context (cf/logging-context)
                    :timeout timeout
                    :cause cause)
             (px/sleep timeout))
 
-          (l/err :hint "unhandled exception" :cause cause))))))
+          (l/err :hint "unhandled exception"
+                 ::l/context (cf/logging-context)
+                 :cause cause))))))
 
 (defn- start-thread!
   [{:keys [::id ::queue ::wrk/tenant] :as cfg}]
@@ -284,6 +293,7 @@
                  :queue queue))
         (catch Throwable cause
           (l/err :hint "unexpected exception"
+                 ::l/context (cf/logging-context)
                  :id id
                  :queue queue
                  :cause cause))

backend/test/backend_tests/email_sending_test.clj

@@ -22,4 +22,4 @@
     (t/is (contains? result :body))
     (t/is (contains? result :to))
     #_(t/is (contains? result :reply-to))
-    (t/is (vector? (:body result)))))
+    (t/is (map? (:body result)))))

backend/test/backend_tests/helpers.clj

@@ -30,6 +30,7 @@
    [app.rpc.commands.files :as files]
    [app.rpc.commands.files-create :as files.create]
    [app.rpc.commands.files-update :as files.update]
+   [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.helpers :as rph]
    [app.util.blob :as blob]
@@ -104,13 +105,8 @@
                      (assoc-in [:app.rpc/methods :app.setup/templates] templates)
                      (dissoc :app.srepl/server
                              :app.http/server
-                             :app.http/router
+                             :app.http/route
-                             :app.auth.oidc.providers/google
-                             :app.auth.oidc.providers/gitlab
-                             :app.auth.oidc.providers/github
-                             :app.auth.oidc.providers/generic
                              :app.setup/templates
-                             :app.auth.oidc/routes
                              :app.http.oauth/handler
                              :app.notifications/handler
                              :app.loggers.mattermost/reporter
@@ -182,23 +178,25 @@
                         :is-demo false}
                        params)]
      (db/run! system
-              (fn [{:keys [::db/conn]}]
+              (fn [{:keys [::db/conn] :as cfg}]
                 (->> params
-                     (cmd.auth/create-profile! conn)
+                     (cmd.auth/create-profile cfg)
-                     (cmd.auth/create-profile-rels! conn)))))))
+                     (cmd.auth/create-profile-rels conn)))))))
 
 (defn create-project*
   ([i params] (create-project* *system* i params))
   ([system i {:keys [profile-id team-id] :as params}]
-   (us/assert uuid? profile-id)
-   (us/assert uuid? team-id)
 
-   (db/run! system
+   (assert (uuid? profile-id))
-            (fn [{:keys [::db/conn]}]
+   (assert (uuid? team-id))
-              (->> (merge {:id (mk-uuid "project" i)
+   (let [timestamp (ct/now)]
-                           :name (str "project" i)}
+     (db/run! system
-                          params)
+              (fn [cfg]
-                   (#'teams/create-project conn))))))
+                (->> (merge {:id (mk-uuid "project" i)
+                             :name (str "project" i)}
+                            params
+                            {::rpc/request-at timestamp})
+                     (#'projects/create-project cfg)))))))
 
 (defn create-file*
   ([i params]
@@ -549,6 +547,44 @@
       (io/copy r sw)
       (.toString sw))))
 
+(defn parse-sse
+  [content]
+  (let [state
+        (reduce (fn [{:keys [events data event id] :as state} line]
+                  (cond
+                    ;; empty line → dispatch event if we have data
+                    (str/blank? line)
+                    (if (seq data)
+                      (-> state
+                          (update :events conj {:event (or event "message")
+                                                :data (-> (str/join "\n" data))})
+                          (assoc :data [] :event nil))
+                      state)
+
+                    ;; comment line (starts with :)
+                    (str/starts-with? line ":")
+                    state
+
+                    :else
+                    (let [[field raw-value] (str/split line #":" 2)
+                          value (some-> raw-value (str/replace #"^ " ""))]
+                      (case field
+                        "data"  (update state :data conj (or value ""))
+                        "event" (assoc state :event value)
+                        ;; ignore retry and unknown fields
+                        state))))
+                {:events [] :data [] :event nil}
+                (str/split content #"\r?\n"))
+
+        ;; handle unterminated last event (no trailing blank line)
+        state (if (seq (:data state))
+                (update state :events conj
+                        {:event (or (:event state) "message")
+                         :data (str/join "\n" (:data state))})
+                state)]
+
+    (:events state)))
+
 (defn consume-sse
   [callback]
   (let [{:keys [::yres/status ::yres/body ::yres/headers] :as response} (callback {})
@@ -558,12 +594,9 @@
     (try
       (px/exec! :virtual #(rcp/write-body-to-stream body nil output))
       (into []
-            (map (fn [event]
+            (map (fn [{:keys [event data]}]
-                   (let [[item1 item2] (re-seq #"(.*): (.*)\n?" event)]
+                   (d/vec2 (keyword event)
-
+                           (tr/decode-str data))))
-                     [(keyword (nth item1 2))
+            (parse-sse (slurp' input)))
-                      (tr/decode-str (nth item2 2))])))
-            (-> (slurp' input)
-                (str/split "\n\n")))
       (finally
         (.close input)))))

backend/test/backend_tests/http_management_test.clj

@@ -22,17 +22,6 @@
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
 
-
-(t/deftest authenticate-method
-  (let [profile  (th/create-profile* 1)
-        token    (#'sess/gen-token th/*system* {:profile-id (:id profile)})
-        request  {:params {:token token}}
-        response (#'mgmt/authenticate th/*system* request)]
-
-    (t/is (= 200 (::yres/status response)))
-    (t/is (= "authentication" (-> response ::yres/body :iss)))
-    (t/is (= (:id profile) (-> response ::yres/body :uid)))))
-
 (t/deftest get-customer-method
   (let [profile  (th/create-profile* 1)
         request  {:params {:id (:id profile)}}
@@ -89,7 +78,3 @@
 
       (let [subs' (-> response ::yres/body :subscription)]
         (t/is (= subs' subs))))))
-
-
-
-

backend/test/backend_tests/http_middleware_access_token_test.clj

@@ -1,57 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns backend-tests.http-middleware-access-token-test
-  (:require
-   [app.db :as db]
-   [app.http.access-token]
-   [app.main :as-alias main]
-   [app.rpc :as-alias rpc]
-   [app.rpc.commands.access-token]
-   [app.tokens :as tokens]
-   [backend-tests.helpers :as th]
-   [clojure.test :as t]
-   [mockery.core :refer [with-mocks]]))
-
-(t/use-fixtures :once th/state-init)
-(t/use-fixtures :each th/database-reset)
-
-(t/deftest soft-auth-middleware
-  (let [profile (th/create-profile* 1)
-        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
-
-        request (volatile! nil)
-        handler (#'app.http.access-token/wrap-soft-auth
-                 (fn [req] (vreset! request req))
-                 th/*system*)]
-
-    (with-mocks [m1 {:target 'app.http.access-token/get-token
-                     :return nil}]
-      (handler {})
-      (t/is (= {} @request)))
-
-    (with-mocks [m1 {:target 'app.http.access-token/get-token
-                     :return (:token token)}]
-      (handler {})
-
-      (let [token-id (get @request :app.http.access-token/id)]
-        (t/is (= token-id (:id token)))))))
-
-(t/deftest authz-middleware
-  (let [profile (th/create-profile* 1)
-        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
-        request (volatile! {})
-        handler (#'app.http.access-token/wrap-authz
-                 (fn [req] (vreset! request req))
-                 th/*system*)]
-
-    (handler nil)
-    (t/is (nil? @request))
-
-    (handler {:app.http.access-token/id (:id token)})
-    (t/is (= #{} (:app.http.access-token/perms @request)))
-    (t/is (= (:id profile) (:app.http.access-token/profile-id @request)))))
-

backend/test/backend_tests/http_middleware_test.clj

@@ -0,0 +1,135 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns backend-tests.http-middleware-test
+  (:require
+   [app.common.time :as ct]
+   [app.db :as db]
+   [app.http :as-alias http]
+   [app.http.access-token]
+   [app.http.middleware :as mw]
+   [app.http.session :as session]
+   [app.main :as-alias main]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.access-token]
+   [app.tokens :as tokens]
+   [backend-tests.helpers :as th]
+   [clojure.test :as t]
+   [mockery.core :refer [with-mocks]]
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
+
+(t/use-fixtures :once th/state-init)
+(t/use-fixtures :each th/database-reset)
+
+(defrecord DummyRequest [headers cookies]
+  yreq/IRequestCookies
+  (get-cookie [_ name]
+    {:value (get cookies name)})
+
+  yreq/IRequest
+  (get-header [_ name]
+    (get headers name)))
+
+(t/deftest auth-middleware-1
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {"authorization" "Token aaaa"} {}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :token token-type))
+      (t/is (= "aaaa" token))
+      (t/is (nil? claims)))))
+
+(t/deftest auth-middleware-2
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {"authorization" "Bearer aaaa"} {}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :bearer token-type))
+      (t/is (= "aaaa" token))
+      (t/is (nil? claims)))))
+
+(t/deftest auth-middleware-3
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {} {"auth-token" "foobar"}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :cookie token-type))
+      (t/is (= "foobar" token))
+      (t/is (nil? claims)))))
+
+(t/deftest shared-key-auth
+  (let [handler (#'app.http.middleware/wrap-shared-key-auth
+                 (fn [req] {::yres/status 200})
+                 "secret-key")]
+
+    (let [response (handler (->DummyRequest {} {}))]
+      (t/is (= 403 (::yres/status response))))
+
+    (let [response (handler (->DummyRequest {"x-shared-key" "secret-key2"} {}))]
+      (t/is (= 403 (::yres/status response))))
+
+    (let [response (handler (->DummyRequest {"x-shared-key" "secret-key"} {}))]
+      (t/is (= 200 (::yres/status response))))))
+
+(t/deftest access-token-authz
+  (let [profile (th/create-profile* 1)
+        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
+        handler (#'app.http.access-token/wrap-authz identity th/*system*)]
+
+    (let [response (handler nil)]
+      (t/is (nil? response)))
+
+    (let [response (handler {::http/auth-data {:type :token :token "foobar" :claims {:tid (:id token)}}})]
+      (t/is (= #{} (:app.http.access-token/perms response)))
+      (t/is (= (:id profile) (:app.http.access-token/profile-id response))))))
+
+(t/deftest session-authz
+  (let [cfg      th/*system*
+        manager  (session/inmemory-manager)
+        profile  (th/create-profile* 1)
+        handler  (-> (fn [req] req)
+                     (#'session/wrap-authz  {::session/manager manager})
+                     (#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
+                                      :cookie (partial session/decode-token cfg)}))
+
+        session  (->> (session/create-session manager {:profile-id (:id profile)
+                                                       :user-agent "user agent"})
+                      (#'session/assign-token cfg))
+
+        response (handler (->DummyRequest {} {"auth-token" (:token session)}))
+
+        {:keys [token claims] token-type :type}
+        (get response ::http/auth-data)]
+
+    (t/is (= :cookie token-type))
+    (t/is (= (:token session) token))
+    (t/is (= "authentication" (:iss claims)))
+    (t/is (= "penpot" (:aud claims)))
+    (t/is (= (:id session) (:sid claims)))
+    (t/is (= (:id profile) (:uid claims)))))

backend/test/backend_tests/rpc_doc_test.clj

@@ -23,7 +23,7 @@
   (smt/check!
    (smt/for [context (->> sg/int
                           (sg/fmap (fn [_]
-                                     (rpc.doc/prepare-openapi-context (::rpc/methods th/*system*)))))]
+                                     (#'rpc.doc/openapi-context (::rpc/methods th/*system*)))))]
      (try
        (json/encode context)
        true

backend/test/backend_tests/rpc_file_snapshot_test.clj

@@ -9,6 +9,7 @@
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
    [app.common.thumbnails :as thc]
+   [app.common.time :as ct]
    [app.common.types.shape :as cts]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -16,6 +17,7 @@
    [app.db.sql :as sql]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -132,9 +134,10 @@
         ;; this will run pending task triggered by deleting user snapshot
         (th/run-pending-tasks!)
 
-        (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+        (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-          ;; delete 2 snapshots and 2 file data entries
+          (let [res (th/run-task! :objects-gc {})]
-          (t/is (= 4 (:processed res))))))))
+            ;; delete 2 snapshots and 2 file data entries
+            (t/is (= 4 (:processed res)))))))))
 
 (t/deftest snapshots-locking
   (let [profile-1 (th/create-profile* 1 {:is-active true})

backend/test/backend_tests/rpc_file_test.clj

@@ -19,6 +19,7 @@
    [app.http :as http]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -142,126 +143,112 @@
           (t/is (= 0 (count result))))))))
 
 (t/deftest file-gc-with-fragments
-  (letfn [(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
+  (let [profile (th/create-profile* 1)
-            (let [params {::th/type :update-file
+        file    (th/create-file* 1 {:profile-id (:id profile)
-                          ::rpc/profile-id profile-id
+                                    :project-id (:default-project-id profile)
-                          :id file-id
+                                    :is-shared false})
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :features cfeat/supported-features
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
-              (:result out)))]
 
-    (let [profile (th/create-profile* 1)
+        page-id  (uuid/random)
-          file    (th/create-file* 1 {:profile-id (:id profile)
+        shape-id (uuid/random)]
-                                      :project-id (:default-project-id profile)
-                                      :is-shared false})
 
-          page-id  (uuid/random)
+    ;; Preventive file-gc
-          shape-id (uuid/random)]
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file) :revn (:revn file)})))
 
-      ;; Preventive file-gc
+    ;; Check the number of fragments before adding the page
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file) :revn (:revn file)})))
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments before adding the page
+    ;; Add page
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+    (update-file!
-        (t/is (= 2 (count rows))))
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-page
+       :name "test"
+       :id page-id}])
 
-      ;; Add page
+    ;; Check the number of fragments before adding the page
-      (update-file!
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-       :file-id (:id file)
+      (t/is (= 3 (count rows))))
-       :profile-id (:id profile)
-       :revn 0
-       :vern 0
-       :changes
-       [{:type :add-page
-         :name "test"
-         :id page-id}])
 
-      ;; Check the number of fragments before adding the page
+    ;; The file-gc should mark for remove unused fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
-        (t/is (= 3 (count rows))))
 
-      ;; The file-gc should mark for remove unused fragments
+    ;; Check the number of fragments
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 5 (count rows)))
+      (t/is (= 3 (count (filterv :deleted-at rows)))))
 
-      ;; Check the number of fragments
+    ;; The objects-gc should remove unused fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+    (let [res (th/run-task! :objects-gc {})]
-        (t/is (= 5 (count rows)))
+      (t/is (= 3 (:processed res))))
-        (t/is (= 3 (count (filterv :deleted-at rows)))))
 
-      ;; The objects-gc should remove unused fragments
+    ;; Check the number of fragments
-      (let [res (th/run-task! :objects-gc {})]
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 3 (:processed res))))
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments
+    ;; Add shape to page that should add a new fragment
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+    (update-file!
-        (t/is (= 2 (count rows))))
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id page-id
+       :id shape-id
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id shape-id
+              :name "image"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :rect})}])
 
-      ;; Add shape to page that should add a new fragment
+    ;; Check the number of fragments
-      (update-file!
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-       :file-id (:id file)
+      (t/is (= 3 (count rows))))
-       :profile-id (:id profile)
-       :revn 0
-       :vern 0
-       :changes
-       [{:type :add-obj
-         :page-id page-id
-         :id shape-id
-         :parent-id uuid/zero
-         :frame-id uuid/zero
-         :components-v2 true
-         :obj (cts/setup-shape
-               {:id shape-id
-                :name "image"
-                :frame-id uuid/zero
-                :parent-id uuid/zero
-                :type :rect})}])
 
-      ;; Check the number of fragments
+    ;; The file-gc should mark for remove unused fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
-        (t/is (= 3 (count rows))))
 
-      ;; The file-gc should mark for remove unused fragments
+    ;; The objects-gc should remove unused fragments
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+    (let [res (th/run-task! :objects-gc {})]
+      (t/is (= 3 (:processed res))))
 
-      ;; The objects-gc should remove unused fragments
+    ;; Check the number of fragments;
-      (let [res (th/run-task! :objects-gc {})]
+    (let [rows (th/db-query :file-data {:file-id (:id file)
-        (t/is (= 3 (:processed res))))
+                                        :type "fragment"
+                                        :deleted-at nil})]
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments;
+    ;; Lets proceed to delete all changes
-      (let [rows (th/db-query :file-data {:file-id (:id file)
+    (th/db-delete! :file-change {:file-id (:id file)})
-                                          :type "fragment"
+    (th/db-delete! :file-data {:file-id (:id file) :type "snapshot"})
-                                          :deleted-at nil})]
-        (t/is (= 2 (count rows))))
 
-      ;; Lets proceed to delete all changes
+    (th/db-update! :file
-      (th/db-delete! :file-change {:file-id (:id file)})
+                   {:has-media-trimmed false}
-      (th/db-delete! :file-data {:file-id (:id file) :type "snapshot"})
+                   {:id (:id file)})
 
-      (th/db-update! :file
+    ;; The file-gc should remove fragments related to changes
-                     {:has-media-trimmed false}
+    ;; snapshots previously deleted.
-                     {:id (:id file)})
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      ;; The file-gc should remove fragments related to changes
+    ;; Check the number of fragments;
-      ;; snapshots previously deleted.
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+      ;; (pp/pprint rows)
+      (t/is (= 4 (count rows)))
+      (t/is (= 2 (count (remove :deleted-at rows)))))
 
-      ;; Check the number of fragments;
+    (let [res (th/run-task! :objects-gc {})]
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 2 (:processed res))))
-        ;; (pp/pprint rows)
-        (t/is (= 4 (count rows)))
-        (t/is (= 2 (count (remove :deleted-at rows)))))
 
-      (let [res (th/run-task! :objects-gc {})]
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 2 (:processed res))))
+      (t/is (= 2 (count rows))))))
-
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 2 (count rows)))))))
 
 (t/deftest file-gc-with-thumbnails
   (letfn [(add-file-media-object [& {:keys [profile-id file-id]}]
@@ -279,20 +266,6 @@
 
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
-              (:result out)))
-
-          (update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-            (let [params {::th/type :update-file
-                          ::rpc/profile-id profile-id
-                          :id file-id
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :features cfeat/supported-features
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
               (:result out)))]
 
     (let [storage (:app.storage/storage th/*system*)
@@ -340,7 +313,7 @@
       ;; freeze because of the deduplication (we have uploaded 2 times
       ;; the same files).
 
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 2 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
@@ -399,14 +372,14 @@
       (th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
       (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
 
-      (let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
+      (let [res (th/run-task! :objects-gc {})]
         ;; this will remove the file change and file data entries for two snapshots
         (t/is (= 4 (:processed res))))
 
       ;; Rerun the file-gc and objects-gc
       (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      (let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
+      (let [res (th/run-task! :objects-gc {})]
         ;; this will remove the file media objects marked as deleted
         ;; on prev file-gc
         (t/is (= 2 (:processed res))))
@@ -414,7 +387,7 @@
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
       ;; them are marked to be deleted
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 0 (:freeze res)))
         (t/is (= 2 (:delete res))))
 
@@ -599,7 +572,7 @@
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
       ;; them are marked to be deleted.
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 0 (:freeze res)))
         (t/is (= 2 (:delete res))))
 
@@ -692,7 +665,7 @@
       ;; because of the deduplication (we have uploaded 2 times the
       ;; same files).
 
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 1 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
@@ -742,7 +715,7 @@
 
       ;; Now that objects-gc have deleted the object thumbnail lets
       ;; execute the touched-gc task
-      (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
+      (let [res (th/run-task! "storage-gc-touched" {})]
         (t/is (= 1 (:freeze res))))
 
       ;; check file media objects
@@ -777,7 +750,7 @@
 
       ;; Now that file-gc have deleted the object thumbnail lets
       ;; execute the touched-gc task
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 1 (:delete res))))
 
       ;; check file media objects
@@ -949,8 +922,9 @@
       (t/is (= 0 (:processed result))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 3 (:processed result))))
+      (let [result (th/run-task! :objects-gc {})]
+        (t/is (= 3 (:processed result)))))
 
     ;; query the list of file libraries of a after hard deletion
     (let [data {::th/type :get-file-libraries
@@ -1161,7 +1135,7 @@
       (th/sleep 300)
 
       ;; run the task
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       ;; check that object thumbnails are still here
       (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1190,7 +1164,7 @@
         (t/is (= 2 (count rows))))
 
       ;; run the task again
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       ;; check that we have all object thumbnails
       (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1253,7 +1227,7 @@
         (t/is (= 2 (count rows)))))
 
     (t/testing "gc task"
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
         (t/is (= 2 (count rows)))
@@ -1300,7 +1274,7 @@
     ;; The FileGC task will schedule an inner taskq
     (th/run-pending-tasks!)
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 2 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
@@ -1394,7 +1368,7 @@
 
     ;; we ensure that once object-gc is passed and marked two storage
     ;; objects to delete
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 2 (:delete res))))
 
@@ -1516,7 +1490,7 @@
         (t/is (some? (not-empty (:objects component))))))
 
     ;; Re-run the file-gc task
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
     (let [row (th/db-get :file {:id (:id file)})]
       (t/is (true? (:has-media-trimmed row))))
 
@@ -1546,7 +1520,7 @@
 
     ;; Now, we have deleted the usage of component if we pass file-gc,
     ;; that component should be deleted
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
     ;; Check that component is properly removed
     (let [data {::th/type :get-file
@@ -1637,8 +1611,8 @@
               :component-id c-id})}])
 
     ;; Run the file-gc on file and library
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
 
     ;; Check that component exists
     (let [data {::th/type :get-file
@@ -1711,7 +1685,7 @@
 
     ;; Now, we have deleted the usage of component if we pass file-gc,
     ;; that component should be deleted
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
 
     ;; Check that component is properly removed
     (let [data {::th/type :get-file
@@ -1860,8 +1834,8 @@
         (t/is (not= (:id fill) (:id fmedia)))))
 
     ;; Run the file-gc on file and library
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
 
     ;; Now proceed to delete file and absorb it
     (let [data {::th/type :delete-file
@@ -1893,3 +1867,204 @@
 
       (t/is (= (:id file-2) (:file-id (get rows 0))))
       (t/is (nil? (:deleted-at (get rows 0)))))))
+
+(t/deftest deleted-files-permanently-delete
+  (let [prof    (th/create-profile* 1 {:is-active true})
+        team-id (:default-team-id prof)
+        proj-id (:default-project-id prof)
+        file-id (uuid/next)
+        now     (ct/inst "2025-10-31T00:00:00Z")]
+
+    (binding [ct/*clock* (clock/fixed now)]
+      (let [data {::th/type :create-file
+                  ::rpc/profile-id (:id prof)
+                  :project-id proj-id
+                  :id file-id
+                  :name "foobar"
+                  :is-shared false
+                  :components-v2 true}
+            out (th/command! data)]
+
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+
+        (let [result (:result out)]
+          (t/is (= (:name data) (:name result)))
+          (t/is (= proj-id (:project-id result)))))
+
+      (let [data {::th/type :delete-file
+                  :id file-id
+                  ::rpc/profile-id (:id prof)}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (t/is (nil? (:result out))))
+
+      ;; get deleted files
+      (let [data {::th/type :get-team-deleted-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [[row1 :as result] (:result out)]
+          (t/is (= 1 (count result)))
+          (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
+
+      (let [data {::th/type :permanently-delete-team-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id
+                  :ids #{file-id}}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [result (:result out)]
+          (t/is (fn? result))
+
+          (let [[ev1 ev2 :as events] (th/consume-sse result)]
+            (t/is (= 2 (count events)))
+            (t/is (= (:ids data) (val ev2)))))
+
+        (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
+          (t/is (= (:deleted-at row) now)))))))
+
+(t/deftest restore-deleted-files
+  (let [prof    (th/create-profile* 1 {:is-active true})
+        team-id (:default-team-id prof)
+        proj-id (:default-project-id prof)
+        file-id (uuid/next)
+        now     (ct/inst "2025-10-31T00:00:00Z")]
+
+    (binding [ct/*clock* (clock/fixed now)]
+      (let [data {::th/type :create-file
+                  ::rpc/profile-id (:id prof)
+                  :project-id proj-id
+                  :id file-id
+                  :name "foobar"
+                  :is-shared false
+                  :components-v2 true}
+            out (th/command! data)]
+
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+
+        (let [result (:result out)]
+          (t/is (= (:name data) (:name result)))
+          (t/is (= proj-id (:project-id result)))))
+
+      (let [data {::th/type :delete-file
+                  :id file-id
+                  ::rpc/profile-id (:id prof)}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (t/is (nil? (:result out))))
+
+      ;; get deleted files
+      (let [data {::th/type :get-team-deleted-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [[row1 :as result] (:result out)]
+          (t/is (= 1 (count result)))
+          (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
+
+      (let [data {::th/type :restore-deleted-team-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id
+                  :ids #{file-id}}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [result (:result out)]
+          (t/is (fn? result))
+
+          (let [events (th/consume-sse result)]
+            ;; (pp/pprint events)
+            (t/is (= 2 (count events)))
+            (t/is (= :end (first (last events))))
+            (t/is (= (:ids data) (last (last events)))))))
+
+      (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
+        (t/is (nil? (:deleted-at row)))))))
+
+
+(t/deftest restore-deleted-files-and-projets
+  (let [profile (th/create-profile* 1 {:is-active true})
+        team-id (:default-team-id profile)
+        now     (ct/inst "2025-10-31T00:00:00Z")]
+
+    (binding [ct/*clock* (clock/fixed now)]
+      (let [project (th/create-project* 1 {:profile-id (:id profile)
+                                           :team-id team-id})
+            file    (th/create-file* 1 {:profile-id (:id profile)
+                                        :project-id (:id project)})
+
+            data    {::th/type :delete-project
+                     :id (:id project)
+                     ::rpc/profile-id (:id profile)}
+            out     (th/command! data)]
+
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (t/is (nil? (:result out)))
+
+        (th/run-pending-tasks!)
+
+        ;; get deleted files
+        (let [data {::th/type :get-team-deleted-files
+                    ::rpc/profile-id (:id profile)
+                    :team-id team-id}
+              out (th/command! data)]
+          ;; (th/print-result! out)
+          (t/is (nil? (:error out)))
+          (let [[row1 :as result] (:result out)]
+            (t/is (= 1 (count result)))
+            (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+            (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+            (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
+
+        ;; Check if project is deleted
+        (let [[row1 :as rows] (th/db-query :project {:id (:id project)})]
+          ;; (pp/pprint rows)
+          (t/is (= 1 (count rows)))
+          (t/is (= (:deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z")))
+
+        ;; Restore files
+        (let [data {::th/type :restore-deleted-team-files
+                    ::rpc/profile-id (:id profile)
+                    :team-id team-id
+                    :ids #{(:id file)}}
+              out (th/command! data)]
+          ;; (th/print-result! out)
+          (t/is (nil? (:error out)))
+          (let [result (:result out)]
+            (t/is (fn? result))
+            (let [events (th/consume-sse result)]
+              ;; (pp/pprint events)
+              (t/is (= 2 (count events)))
+              (t/is (= :end (first (last events))))
+              (t/is (= (:ids data) (last (last events)))))))
+
+
+        (let [[row1 :as rows] (th/db-query :file {:project-id (:id project)})]
+          ;; (pp/pprint rows)
+          (t/is (= 1 (count rows)))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (nil? (:deleted-at row1))))
+
+
+        ;; Check if project is restored
+        (let [[row1 :as rows] (th/db-query :project {:id (:id project)})]
+          ;; (pp/pprint rows)
+          (t/is (= 1 (count rows)))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (nil? (:deleted-at row1))))))))