✨ Support undo and redo on text

* ♻️ Copy shorthands using user selected color space

* ♻️ Add tests to ensure color space changes affect all properties

.github/workflows/build-bundle.yml

@@ -84,8 +84,10 @@ jobs:
         uses: mattermost/action-mattermost-notify@master
         with:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
           TEXT: |
-            ❌ *[PENPOT] Error during the execution of the job*
+            ❌ 📦 *[PENPOT] Error building penpot bundles.*
             📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
                Bundle version: `${{ steps.vars.outputs.bundle_version }}`
             🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/build-docker.yml

@@ -34,18 +34,26 @@ jobs:
           echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
 
       - name: Download Penpot Bundles
+        id: bundles
         env:
           FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
         run: |
+          tmp=$(aws s3api head-object \
+            --bucket ${{ secrets.S3_BUCKET }} \
+            --key "$FILE_NAME" \
+            --query 'Metadata."bundle-version"' \
+            --output text)
+          echo "bundle_version=$tmp" >> $GITHUB_OUTPUT
           pushd docker/images
           aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
           unzip $FILE_NAME > /dev/null
           mv penpot/backend bundle-backend
           mv penpot/frontend bundle-frontend
           mv penpot/exporter bundle-exporter
+          mv penpot/storybook bundle-storybook
           popd
 
       - name: Set up Docker Buildx
@@ -58,6 +66,18 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images:
+            frontend
+            backend
+            exporter
+            storybook
+          labels: |
+            bundle_version=${{ steps.bundles.outputs.bundle_version }}
+
       - name: Build and push Backend Docker image
         uses: docker/build-push-action@v6
         env:
@@ -69,6 +89,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
 
@@ -83,6 +104,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
 
@@ -97,5 +119,34 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
           cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Build and push Storybook Docker image
+        uses: docker/build-push-action@v6
+        env:
+          DOCKER_IMAGE: 'storybook'
+          BUNDLE_PATH: './bundle-storybook'
+        with:
+          context: ./docker/images/
+          file: ./docker/images/Dockerfile.storybook
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🐳 *[PENPOT] Error building penpot docker images.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            📦 Bundle: `${{ steps.bundles.outputs.bundle_version }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

.github/workflows/commit-checker.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Check Commit Type
         uses: gsactions/commit-message-checker@v2
         with:
-          pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s["A-Z].*[^.]$'
+          pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert).+[^.])$'
           flags: 'gm'
           error: 'Commit should match CONTRIBUTING.md guideline'
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

.github/workflows/release.yml

@@ -37,36 +37,43 @@ jobs:
           ref: ${{ steps.vars.outputs.gh_ref }}
 
       # --- Publicly release the docker images ---
-      - name: Login to private registry
-        uses: docker/login-action@v3
+      - name: Configure ECR credentials
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          registry: ${{ secrets.DOCKER_REGISTRY }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
+          aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
+          aws-region: ${{ secrets.AWS_REGION }}
 
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.PUB_DOCKER_USERNAME }}
-          password: ${{ secrets.PUB_DOCKER_PASSWORD }}
-
-      - name: Publish docker images to DockerHub
-        env:
-          TAG: ${{ steps.vars.outputs.gh_ref }}
-          REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
-          HUB: ${{ secrets.PUB_DOCKER_HUB }}
+      - name: Install Skopeo
         run: |
-          IMAGES=("frontend" "backend" "exporter")
-          EXTRA_TAGS=("main" "latest")
+          sudo apt-get update -y
+          sudo apt-get install -y skopeo
+
+      - name: Copy images from AWS ECR to Docker Hub
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+          PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
+          PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
+          TAG: ${{ steps.vars.outputs.gh_ref }}
+        run: |
+          aws ecr get-login-password --region $AWS_REGION | \
+            skopeo login --username AWS --password-stdin \
+            $DOCKER_REGISTRY
+
+          echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io
+
+          IMAGES=("frontend" "backend" "exporter" "storybook")
 
           for image in "${IMAGES[@]}"; do
-            docker pull "$REGISTRY/penpotapp/$image:$TAG"
-            docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$TAG"
-            docker push "penpotapp/$image:$TAG"
+            skopeo copy --all \
+              docker://$DOCKER_REGISTRY/$image:$TAG \
+              docker://docker.io/penpotapp/$image:$TAG
 
-            for tag in "${EXTRA_TAGS[@]}"; do
-              docker tag "$REGISTRY/penpotapp/$image:$TAG" "penpotapp/$image:$tag"
-              docker push "penpotapp/$image:$tag"
+            for alias in main latest; do
+              skopeo copy --all \
+                docker://$DOCKER_REGISTRY/$image:$TAG \
+                docker://docker.io/penpotapp/$image:$alias
             done
           done
 
@@ -93,3 +100,15 @@ jobs:
           tag_name: ${{ steps.vars.outputs.gh_ref }}
           name: ${{ steps.vars.outputs.gh_ref }}
           body: ${{ steps.extract_release_notes.outputs.release_notes }}
+
+      - name: Notify Mattermost
+        if: failure()
+        uses: mattermost/action-mattermost-notify@master
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
+          MATTERMOST_CHANNEL: bot-alerts-cicd
+          TEXT: |
+            ❌ 🚀 *[PENPOT] Error releasing penpot.*
+            📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
+            🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            @infra

CHANGES.md

@@ -4,18 +4,76 @@
 
 ### :boom: Breaking changes & Deprecations
 
+#### Backend RPC API changes
+
+The backend RPC API URLS are changed from `/api/rpc/command/<name>` to
+`/api/main/methods/<name>` (the previou PATH is preserved for backward
+compatibility; however, if you are a user of this API, it is strongly
+recommended that you adapt your code to use the new PATH.
+
+
+#### Updated SSO Callback URL
+
+The OAuth / Single Sign-On (SSO) callback endpoint has changed to
+align with the new OpenID Connect (OIDC) implementation.
+
+Old callback URL:
+
+```
+https://<your_domain>/api/auth/oauth/<oauth_provider>/callback
+```
+
+New callback URL:
+
+```
+https://<your_domain>/api/auth/oidc/callback
+```
+
+**Action required:**
+
+If you have SSO/Social-Auth configured on your on-premise instance,
+the following actions are required before update:
+
+Update your OAuth or SSO provider configuration (e.g., Okta, Google,
+Azure AD, etc.) to use the new callback URL.  Failure to update may
+result in authentication failures after upgrading.
+
+**Reason for change:**
+
+This update standardizes all authentication flows under the single URL
+and makis it more modular, enabling the ability to configure SSO auth
+provider dinamically.
+
+
 ### :rocket: Epics and highlights
 
 ### :heart: Community contributions (Thank you!)
 
 ### :sparkles: New features & Enhancements
 
+- Select boards to export as PDF [Taiga #12320](https://tree.taiga.io/project/penpot/issue/12320)
+- Toggle for switching boolean property values [Taiga #12341](https://tree.taiga.io/project/penpot/us/12341)
+- Add auth flow changes [Taiga #12333](https://tree.taiga.io/project/penpot/us/12333)
+
 ### :bug: Bugs fixed
 
+- Fix text line-height values are wrong [Taiga #12252](https://tree.taiga.io/project/penpot/issue/12252)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
 - Fix pan cursor not disabling viewport guides [Github #6985](https://github.com/penpot/penpot/issues/6985)
 - Fix viewport resize on locked shapes [Taiga #11974](https://tree.taiga.io/project/penpot/issue/11974)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix on copy instance inside a components chain touched are missing [Taiga #12371](https://tree.taiga.io/project/penpot/issue/12371)
+- Fix problem with multiple selection and shadows [Github #7437](https://github.com/penpot/penpot/issues/7437)
+- Fix search shortcut [Taiga #10265](https://tree.taiga.io/project/penpot/issue/10265)
+- Fix shortcut conflict in text editor (increase/decrease font size vs word selection)
+- Fix problem with plugins generating code for pages different than current one [Taiga #12312](https://tree.taiga.io/project/penpot/issue/12312)
+- Fix input confirmation behavior is not uniform [Taiga #12294](https://tree.taiga.io/project/penpot/issue/12294)
 
-## 2.11.0 (Unreleased)
+## 2.11.1
+
+- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
+
+## 2.11.0
 
 ### :boom: Breaking changes & Deprecations
 
@@ -43,10 +101,6 @@
   services which use netty internally (redis connection, S3 SDK client). This
   configuration is not very commonly used so don't expected real impact on any user.
 
-### :rocket: Epics and highlights
-
-### :heart: Community contributions (Thank you!)
-
 ### :sparkles: New features & Enhancements
 
 - New composite token: Typography [Taiga #10200](https://tree.taiga.io/project/penpot/us/10200)
@@ -56,6 +110,7 @@
 - Alternative ways of creating variants - Button Viewport [Taiga #11931](https://tree.taiga.io/project/penpot/us/11931)
 - Reorder properties for a component [Taiga #10225](https://tree.taiga.io/project/penpot/us/10225)
 - File Data storage layout refactor [Github #7345](https://github.com/penpot/penpot/pull/7345)
+- Make several queries optimization on comment threads [Github #7506](https://github.com/penpot/penpot/pull/7506)
 
 ### :bug: Bugs fixed
 
@@ -70,7 +125,29 @@
 - Fix auto-width changes to fixed when switching variants [Taiga #12172](https://tree.taiga.io/project/penpot/issue/12172)
 - Fix component number has no singular translation string [Taiga #12106](https://tree.taiga.io/project/penpot/issue/12106)
 - Fix adding/removing identical text fills [Taiga #12287](https://tree.taiga.io/project/penpot/issue/12287)
-
+- Fix scroll on the inspect tab [Taiga #12293](https://tree.taiga.io/project/penpot/issue/12293)
+- Fix lock proportion tooltip [Taiga #12326](https://tree.taiga.io/project/penpot/issue/12326)
+- Fix internal Error when selecting a set by name in the token theme editor [Taiga #12310](https://tree.taiga.io/project/penpot/issue/12310)
+- Fix drag & drop functionality is swapping instead or reordering [Taiga #12254](https://tree.taiga.io/project/penpot/issue/12254)
+- Fix variants not syncronizing tokens on switch [Taiga #12290](https://tree.taiga.io/project/penpot/issue/12290)
+- Fix incorrect behavior of Alt + Drag for variants [Taiga #12309](https://tree.taiga.io/project/penpot/issue/12309)
+- Fix text override is lost after switch [Taiga #12269](https://tree.taiga.io/project/penpot/issue/12269)
+- Fix exporting a board crashing the app [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12384)
+- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
+- Fix selected colors not showing colors from children shapes in multiple selection [Taiga #12384](https://tree.taiga.io/project/penpot/issue/12385)
+- Fix scrollbar issue in design tab [Taiga #12367](https://tree.taiga.io/project/penpot/issue/12367)
+- Fix library update notificacions showing when they should not [Taiga #12397](https://tree.taiga.io/project/penpot/issue/12397)
+- Fix remove flex button doesn’t work within variant [Taiga #12314](https://tree.taiga.io/project/penpot/issue/12314)
+- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
+- Fix problem with certain text input in some editable labels (pages, components, tokens...) being in conflict with the drag/drop functionality [Taiga #12316](https://tree.taiga.io/project/penpot/issue/12316)
+- Fix not controlled theme renaming [Taiga #12411](https://tree.taiga.io/project/penpot/issue/12411)
+- Fix paste without selection sends the new element in the back [Taiga #12382](https://tree.taiga.io/project/penpot/issue/12382)
+- Fix options button does not work for comments created in the lower part of the screen [Taiga #12422](https://tree.taiga.io/project/penpot/issue/12422)
+- Fix problem when checking usage with removed teams [Taiga #12442](https://tree.taiga.io/project/penpot/issue/12442)
+- Fix focus mode persisting across page/file navigation [Taiga #12469](https://tree.taiga.io/project/penpot/issue/12469)
+- Fix shadow color validation [Github #7705](https://github.com/penpot/penpot/pull/7705)
+- Fix exception on selection blend-mode using keyboard [Github #7710](https://github.com/penpot/penpot/pull/7710)
+- Fix crash when using decimal (floating-point) values for X/Y or width/height [Taiga #12543](https://tree.taiga.io/project/penpot/issue/12543)
 
 ## 2.10.1
 
@@ -78,12 +155,10 @@
 
 - Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
 
-
 ### :bug: Bugs fixed
 
 - Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
 
-
 ## 2.10.0
 
 ### :rocket: Epics and highlights
@@ -99,7 +174,7 @@
 - Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
 - Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
 - Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
-- New Number token - unitless values  [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
+- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
 - New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
 - New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
 - New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
@@ -180,7 +255,6 @@
 - Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
 - Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
 
-
 ### :bug: Bugs fixed
 
 - Copying font size does not copy the unit [Taiga #11143](https://tree.taiga.io/project/penpot/issue/11143)

backend/deps.edn

@@ -28,8 +28,8 @@
   com.google.guava/guava {:mvn/version "33.4.8-jre"}
 
   funcool/yetti
-  {:git/tag "v11.6"
-   :git/sha "94dc017"
+  {:git/tag "v11.8"
+   :git/sha "1d1b33f"
    :git/url "https://github.com/funcool/yetti.git"
    :exclusions [org.slf4j/slf4j-api]}
 

backend/dev/user.clj

@@ -27,6 +27,7 @@
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]

backend/resources/app/email/feedback/en.html

@@ -8,38 +8,41 @@
   <body>
     <p>
       <strong>Feedback from:</strong><br />
-      {% if profile %}
-        <span>
-          <span>Name: </span>
-          <span><code>{{profile.fullname|abbreviate:25}}</code></span>
-        </span>
-        <br />
-
-        <span>
-          <span>Email: </span>
-          <span>{{profile.email}}</span>
-        </span>
-        <br />
-
-        <span>
-          <span>ID: </span>
-          <span><code>{{profile.id}}</code></span>
-        </span>
-      {% else %}
-        <span>
-          <span>Email: </span>
-          <span>{{profile.email}}</span>
-        </span>
-      {% endif %}
+      <span>
+        <span>Name: </span>
+        <span><code>{{profile.fullname|abbreviate:25}}</code></span>
+      </span>
+      <br />
+      <span>
+        <span>Email: </span>
+        <span>{{profile.email}}</span>
+      </span>
+      <br />
+      <span>
+        <span>ID: </span>
+        <span><code>{{profile.id}}</code></span>
+      </span>
     </p>
     <p>
       <strong>Subject:</strong><br />
-      <span>{{subject|abbreviate:300}}</span>
+      <span>{{feedback-subject|abbreviate:300}}</span>
     </p>
 
+    <p>
+      <strong>Type:</strong><br />
+      <span>{{feedback-type|abbreviate:300}}</span>
+    </p>
+
+    {% if feedback-error-href %}
+    <p>
+      <strong>Error HREF:</strong><br />
+      <span>{{feedback-error-href|abbreviate:500}}</span>
+    </p>
+    {% endif %}
+
     <p>
       <strong>Message:</strong><br />
-      {{content|linebreaks-br|safe}}
+      {{feedback-content|linebreaks-br}}
     </p>
   </body>
 </html>

backend/resources/app/email/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{subject}}
+[PENPOT FEEDBACK]: {{feedback-subject}}

backend/resources/app/email/feedback/en.txt

@@ -1,9 +1,10 @@
-{% if profile %}
-Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
-{% else %}
-Feedback from: {{email}}
-{% endif %}
+From: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+Subject: {{feedback-subject}}
+Type: {{feedback-type}}
+{%- if feedback-error-href %}
+HREF: {{feedback-error-href}}
+{% endif -%}
 
-Subject: {{subject}}
+Message:
 
-{{content}}
+{{feedback-content}}

backend/resources/app/templates/api-doc.tmpl

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="robots" content="noindex,nofollow">
     <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>Builtin API Documentation - Penpot</title>
+    <title>{{label|upper}} API Documentation</title>
 
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
@@ -19,7 +19,7 @@
   <body>
     <main>
       <header>
-        <h1>Penpot API Documentation (v{{version}})</h1>
+        <h1>{{label|upper}}: API Documentation (v{{version}})</h1>
         <small class="menu">
           [
           <nav>
@@ -31,9 +31,10 @@
       </header>
       <section class="doc-content">
         <h2>INTRODUCTION</h2>
-        <p>This documentation is intended to be a general overview of the penpot RPC API.
-          If you prefer, you can use <a href="/api/openapi.json">OpenAPI</a>
-          and/or <a href="/api/openapi">SwaggerUI</a> as alternative.</p>
+        <p>This documentation is intended to be a general overview of
+          the {{label}} API.  If you prefer, you can
+          use <a href="{{openapi}}">Swagger/OpenAPI</a> as
+          alternative.</p>
 
         <h2>GENERAL NOTES</h2>
 
@@ -43,7 +44,7 @@
         that starts with <b>get-</b> in the name, can use GET HTTP
         method which in many cases benefits from the HTTP cache.</p>
 
-
+        {% block auth-section %}
         <h3>Authentication</h3>
         <p>The penpot backend right now offers two way for authenticate the request:
         <b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
@@ -56,9 +57,10 @@
         <p>The access token can be obtained on the appropriate section on profile settings
         and it should be provided using <b>`Authorization`</b> header with <b>`Token
         &lt;token-string&gt;`</b> value.</p>
+        {% endblock %}
 
         <h3>Content Negotiation</h3>
-        <p>The penpot API by default operates indistinctly with: <b>`application/json`</b>
+        <p>This API operates indistinctly with: <b>`application/json`</b>
         and <b>`application/transit+json`</b> content types. You should specify the
         desired content-type on the <b>`Accept`</b> header, the transit encoding is used
         by default.</p>
@@ -75,13 +77,16 @@
         standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
         API</a></p>
 
+        {% block limits-section %}
         <h3>Limits</h3>
         <p>The rate limit work per user basis (this means that different api keys share
         the same rate limit). For now the limits are not documented because we are
         studying and analyzing the data. As a general rule, it should not be abused, if an
         abusive use is detected, we will proceed to block the user's access to the
         API.</p>
+        {% endblock %}
 
+        {% block webhooks-section %}
         <h3>Webhooks</h3>
         <p>All methods that emit webhook events are marked with flag <b>WEBHOOK</b>, the
         data structure defined on each method represents the <i>payload</i> of the
@@ -97,9 +102,11 @@
   "profileId": "db601c95-045f-808b-8002-361312e63531"
 }
         </pre>
+        {% endblock %}
+
       </section>
       <section class="rpc-doc-content">
-        <h2>RPC METHODS REFERENCE:</h2>
+        <h2>METHODS REFERENCE:</h2>
         <ul class="rpc-items">
           {% for item in methods %}
             {% include "app/templates/api-doc-entry.tmpl" with item=item %}

backend/resources/app/templates/main-api-doc.tmpl

@@ -0,0 +1 @@
+{% extends "app/templates/api-doc.tmpl" %}

backend/resources/app/templates/management-api-doc.tmpl

@@ -0,0 +1,10 @@
+{% extends "app/templates/api-doc.tmpl" %}
+
+{% block auth-section %}
+{% endblock %}
+
+{% block limits-section %}
+{% endblock %}
+
+{% block webhooks-section %}
+{% endblock %}

backend/resources/app/templates/openapi.tmpl

@@ -7,7 +7,7 @@
     name="description"
     content="SwaggerUI"
   />
-  <title>PENPOT Swagger UI</title>
+  <title>{{label|upper}} API</title>
   <style>{{swagger-css|safe}}</style>
 </head>
 <body>
@@ -16,7 +16,7 @@
 <script>
   window.onload = () => {
     window.ui = SwaggerUIBundle({
-      url: '{{public-uri}}/api/openapi.json',
+      url: '{{uri}}',
       dom_id: '#swagger-ui',      
       presets: [
         SwaggerUIBundle.presets.apis,

backend/scripts/_env

@@ -7,12 +7,12 @@ export PENPOT_HOST=devenv
 
 export PENPOT_FLAGS="\
        $PENPOT_FLAGS \
-       enable-login-with-ldap \
        enable-login-with-password
-       enable-login-with-oidc \
-       enable-login-with-google \
-       enable-login-with-github \
-       enable-login-with-gitlab \
+       disable-login-with-ldap \
+       disable-login-with-oidc \
+       disable-login-with-google \
+       disable-login-with-github \
+       disable-login-with-gitlab \
        enable-backend-worker \
        enable-backend-asserts \
        disable-feature-fdata-pointer-map \
@@ -20,6 +20,7 @@ export PENPOT_FLAGS="\
        enable-audit-log \
        enable-transit-readable-response \
        enable-demo-users \
+       enable-user-feedback \
        disable-secure-session-cookies \
        enable-smtp \
        enable-prepl-server \
@@ -46,6 +47,8 @@ export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
 # Setup default multipart upload size to 300MiB
 export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
 
+export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com"
+
 export AWS_ACCESS_KEY_ID=penpot-devenv
 export AWS_SECRET_ACCESS_KEY=penpot-devenv
 export PENPOT_OBJECTS_STORAGE_BACKEND=s3

backend/src/app/binfile/common.clj

@@ -550,7 +550,7 @@
   [cfg data file-id]
   (let [library-ids (get-libraries cfg [file-id])]
     (reduce (fn [data library-id]
-              (if-let [library (get-file cfg library-id)]
+              (if-let [library (get-file cfg library-id :include-deleted? true)]
                 (ctf/absorb-assets data (:data library))
                 data))
             data
@@ -749,7 +749,7 @@
           l.version
      FROM libs AS l
     INNER JOIN project AS p ON (p.id = l.project_id)
-    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+    WHERE l.deleted_at IS NULL;")
 
 (defn get-file-libraries
   [conn file-id]

backend/src/app/binfile/v3.clj

@@ -228,6 +228,7 @@
     (db/tx-run! cfg (fn [cfg]
                       (cond-> (bfc/get-file cfg file-id
                                             {:realize? true
+                                             :include-deleted? true
                                              :lock-for-update? true})
                         detach?
                         (-> (ctf/detach-external-references file-id)
@@ -285,14 +286,12 @@
 
     (let [file (cond-> (select-keys file bfc/file-attrs)
                  (:options data)
-                 (assoc :options (:options data))
+                 (assoc :options (:options data)))
 
-                 :always
-                 (dissoc :data))
-
-          file (cond-> file
-                 :always
-                 (encode-file))
+          file (-> file
+                   (dissoc :data)
+                   (dissoc :deleted-at)
+                   (encode-file))
 
           path (str "files/" file-id ".json")]
       (write-entry! output path file))

backend/src/app/config.clj

@@ -47,6 +47,7 @@
    :auto-file-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
+
    :host "localhost"
    :tenant "default"
 
@@ -57,6 +58,8 @@
    :objects-storage-backend "fs"
    :objects-storage-fs-directory "assets"
 
+   :auth-token-cookie-name "auth-token"
+
    :assets-path "/internal/assets/"
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
@@ -90,7 +93,7 @@
     [:secret-key {:optional true} :string]
 
     [:tenant {:optional false} :string]
-    [:public-uri {:optional false} :string]
+    [:public-uri {:optional false} ::sm/uri]
     [:host {:optional false} :string]
 
     [:http-server-port {:optional true} ::sm/int]
@@ -165,7 +168,7 @@
     [:google-client-id {:optional true} :string]
     [:google-client-secret {:optional true} :string]
     [:oidc-client-id {:optional true} :string]
-    [:oidc-user-info-source {:optional true} :keyword]
+    [:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
     [:oidc-client-secret {:optional true} :string]
     [:oidc-base-uri {:optional true} :string]
     [:oidc-token-uri {:optional true} :string]
@@ -319,5 +322,9 @@
   ([key default]
    (c/get config key default)))
 
+(defn logging-context
+  []
+  {:version/backend (:full version)})
+
 ;; Set value for all new threads bindings.
 (alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -704,6 +704,12 @@
   (and (sql-exception? cause)
        (= "40001" (.getSQLState ^java.sql.SQLException cause))))
 
+(defn duplicate-key-error?
+  [cause]
+  (and (sql-exception? cause)
+       (= "23505" (.getSQLState ^java.sql.SQLException cause))))
+
+
 (extend-protocol jdbc.prepare/SettableParameter
   clojure.lang.Keyword
   (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]

backend/src/app/email.clj

@@ -7,6 +7,7 @@
 (ns app.email
   "Main api for send emails."
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
@@ -93,36 +94,44 @@
                headers)))
 
 (defn- assign-body
-  [^MimeMessage mmsg {:keys [body charset] :or {charset "utf-8"}}]
-  (let [mpart (MimeMultipart. "mixed")]
+  [^MimeMessage mmsg {:keys [body charset attachments] :or {charset "utf-8"}}]
+  (let [mixed-mpart  (MimeMultipart. "mixed")]
     (cond
       (string? body)
-      (let [bpart (MimeBodyPart.)]
-        (.setContent bpart ^String body (str "text/plain; charset=" charset))
-        (.addBodyPart mpart bpart))
-
-      (vector? body)
-      (let [mmp (MimeMultipart. "alternative")
-            mbp (MimeBodyPart.)]
-        (.addBodyPart mpart mbp)
-        (.setContent mbp mmp)
-        (doseq [item body]
-          (let [mbp (MimeBodyPart.)]
-            (.setContent mbp
-                         ^String (:content item)
-                         ^String (str (:type item "text/plain") "; charset=" charset))
-            (.addBodyPart mmp mbp))))
+      (let [text-part (MimeBodyPart.)]
+        (.setText text-part ^String body ^String charset)
+        (.addBodyPart mixed-mpart text-part))
 
       (map? body)
-      (let [bpart (MimeBodyPart.)]
-        (.setContent bpart
-                     ^String (:content body)
-                     ^String (str (:type body "text/plain") "; charset=" charset))
-        (.addBodyPart mpart bpart))
+      (let [content-part      (MimeBodyPart.)
+            alternative-mpart (MimeMultipart. "alternative")]
+
+        (when-let [content (get body "text/html")]
+          (let [html-part (MimeBodyPart.)]
+            (.setContent html-part ^String content
+                         (str "text/html; charset=" charset))
+            (.addBodyPart alternative-mpart html-part)))
+
+        (when-let [content (get body "text/plain")]
+          (let [text-part (MimeBodyPart.)]
+            (.setText text-part ^String content ^String charset)
+            (.addBodyPart alternative-mpart text-part)))
+
+        (.setContent content-part alternative-mpart)
+        (.addBodyPart mixed-mpart content-part))
 
       :else
-      (throw (ex-info "Unsupported type" {:body body})))
-    (.setContent mmsg mpart)
+      (throw (IllegalArgumentException. "invalid email body provided")))
+
+    (doseq [[name content] attachments]
+
+      (prn "attachment" name)
+      (let [attachment-part (MimeBodyPart.)]
+        (.setFileName attachment-part ^String name)
+        (.setContent attachment-part ^String content (str "text/plain; charset=" charset))
+        (.addBodyPart mixed-mpart attachment-part)))
+
+    (.setContent mmsg mixed-mpart)
     mmsg))
 
 (defn- opts->props
@@ -210,24 +219,26 @@
       (ex/raise :type :internal
                 :code :missing-email-templates))
     {:subject subj
-     :body (into
-            [{:type "text/plain"
-              :content text}]
-            (when html
-              [{:type "text/html"
-                :content html}]))}))
+     :body (d/without-nils
+            {"text/plain" text
+             "text/html" html})}))
 
-(def ^:private schema:context
-  [:map
+(def ^:private schema:params
+  [:map {:title "Email Params"}
    [:to [:or ::sm/email [::sm/vec ::sm/email]]]
    [:reply-to {:optional true} ::sm/email]
    [:from {:optional true} ::sm/email]
    [:lang {:optional true} ::sm/text]
+   [:subject {:optional true} ::sm/text]
    [:priority {:optional true} [:enum :high :low]]
-   [:extra-data {:optional true} ::sm/text]])
+   [:extra-data {:optional true} ::sm/text]
+   [:body {:optional true}
+    [:or :string [:map-of :string :string]]]
+   [:attachments {:optional true}
+    [:map-of :string :string]]])
 
-(def ^:private check-context
-  (sm/check-fn schema:context))
+(def ^:private check-params
+  (sm/check-fn schema:params))
 
 (defn template-factory
   [& {:keys [id schema]}]
@@ -235,9 +246,9 @@
   (let [check-fn (if schema
                    (sm/check-fn schema)
                    (constantly nil))]
-    (fn [context]
-      (let [context (-> context check-context check-fn)
-            email   (build-email-template id context)]
+    (fn [params]
+      (let [params (-> params check-params check-fn)
+            email  (build-email-template id params)]
         (when-not email
           (ex/raise :type :internal
                     :code :email-template-does-not-exists
@@ -245,35 +256,40 @@
                     :template-id id))
 
         (cond-> (assoc email :id (name id))
-          (:extra-data context)
-          (assoc :extra-data (:extra-data context))
+          (:extra-data params)
+          (assoc :extra-data (:extra-data params))
 
-          (:from context)
-          (assoc :from (:from context))
+          (seq (:attachments params))
+          (assoc :attachments (:attachments params))
 
-          (:reply-to context)
-          (assoc :reply-to (:reply-to context))
+          (:from params)
+          (assoc :from (:from params))
 
-          (:to context)
-          (assoc :to (:to context)))))))
+          (:reply-to params)
+          (assoc :reply-to (:reply-to params))
+
+          (:to params)
+          (assoc :to (:to params)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PUBLIC HIGH-LEVEL API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn render
-  [email-factory context]
-  (email-factory context))
+  [email-factory params]
+  (email-factory params))
 
 (defn send!
   "Schedule an already defined email to be sent using asynchronously
   using worker task."
-  [{:keys [::conn ::factory] :as context}]
+  [{:keys [::conn ::factory] :as params}]
   (assert (db/connectable? conn) "expected a valid database connection or pool")
 
   (let [email (if factory
-                (factory context)
-                (dissoc context ::conn))]
+                (factory params)
+                (-> params
+                    (dissoc params)
+                    (check-params)))]
     (wrk/submit! {::wrk/task :sendmail
                   ::wrk/delay 0
                   ::wrk/max-retries 4
@@ -343,8 +359,10 @@
 
 (def ^:private schema:feedback
   [:map
-   [:subject ::sm/text]
-   [:content ::sm/text]])
+   [:feedback-subject ::sm/text]
+   [:feedback-type ::sm/text]
+   [:feedback-content ::sm/text]
+   [:profile :map]])
 
 (def user-feedback
   "A profile feedback email."

backend/src/app/http.clj

@@ -25,7 +25,6 @@
    [app.main :as-alias main]
    [app.metrics :as mtx]
    [app.rpc :as-alias rpc]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [integrant.core :as ig]
    [reitit.core :as r]
@@ -149,7 +148,6 @@
   [:map
    [::ws/routes schema:routes]
    [::rpc/routes schema:routes]
-   [::rpc.doc/routes schema:routes]
    [::oidc/routes schema:routes]
    [::assets/routes schema:routes]
    [::debug/routes schema:routes]
@@ -171,8 +169,9 @@
                       [sec/sec-fetch-metadata]
                       [mw/params]
                       [mw/format-response]
-                      [session/soft-auth cfg]
-                      [actoken/soft-auth cfg]
+                      [mw/auth {:bearer (partial session/decode-token cfg)
+                                :cookie (partial session/decode-token cfg)
+                                :token  (partial actoken/decode-token cfg)}]
                       [mw/parse-request]
                       [mw/errors errors/handle]
                       [mw/restrict-methods]]}
@@ -188,9 +187,5 @@
       (::mgmt/routes cfg)]
 
      (::ws/routes cfg)
-
-     ["/api" {:middleware [[mw/cors]
-                           [sec/client-header-check]]}
-      (::oidc/routes cfg)
-      (::rpc.doc/routes cfg)
-      (::rpc/routes cfg)]]]))
+     (::oidc/routes cfg)
+     (::rpc/routes cfg)]]))

backend/src/app/http/access_token.clj

@@ -9,23 +9,19 @@
    [app.common.logging :as l]
    [app.config :as cf]
    [app.db :as db]
+   [app.http :as-alias http]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
-   [app.tokens :as tokens]
-   [yetti.request :as yreq]))
+   [app.tokens :as tokens]))
 
-(def header-re #"(?i)^Token\s+(.*)")
-
-(defn get-token
-  [request]
-  (some->> (yreq/get-header request "authorization")
-           (re-matches header-re)
-           (second)))
-
-(defn- decode-token
+(defn decode-token
   [cfg token]
-  (when token
-    (tokens/verify cfg {:token token :iss "access-token"})))
+  (try
+    (tokens/verify cfg {:token token :iss "access-token"})
+    (catch Throwable cause
+      (l/trc :hint "exception on decoding token"
+             :token token
+             :cause cause))))
 
 (def sql:get-token-data
   "SELECT perms, profile_id, expires_at
@@ -35,47 +31,28 @@
            OR (expires_at > now()));")
 
 (defn- get-token-data
-  [pool token-id]
+  [pool claims]
   (when-not (db/read-only? pool)
-    (some-> (db/exec-one! pool [sql:get-token-data token-id])
-            (update :perms db/decode-pgarray #{}))))
-
-(defn- wrap-soft-auth
-  "Soft Authentication, will be executed synchronously on the undertow
-  worker thread."
-  [handler cfg]
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token cfg token)]
-                (cond-> request
-                  (map? claims)
-                  (assoc ::id (:tid claims))))
-              (catch Throwable cause
-                (l/trace :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
+    (when-let [token-id (get claims :tid)]
+      (some-> (db/exec-one! pool [sql:get-token-data token-id])
+              (update :perms db/decode-pgarray #{})))))
 
 (defn- wrap-authz
-  "Authorization middleware, will be executed synchronously on vthread."
   [handler {:keys [::db/pool]}]
   (fn [request]
-    (let [{:keys [perms profile-id expires-at]} (some->> (::id request) (get-token-data pool))]
-      (handler (cond-> request
-                 (some? perms)
-                 (assoc ::perms perms)
-                 (some? profile-id)
-                 (assoc ::profile-id profile-id)
-                 (some? expires-at)
-                 (assoc ::expires-at expires-at))))))
+    (let [{:keys [type claims]} (get request ::http/auth-data)]
+      (if (= :token type)
+        (let [{:keys [perms profile-id expires-at]} (some->> claims (get-token-data pool))]
+          ;; FIXME: revisit this, this data looks unused
+          (handler (cond-> request
+                     (some? perms)
+                     (assoc ::perms perms)
+                     (some? profile-id)
+                     (assoc ::profile-id profile-id)
+                     (some? expires-at)
+                     (assoc ::expires-at expires-at))))
 
-(def soft-auth
-  {:name ::soft-auth
-   :compile (fn [& _]
-              (when (contains? cf/flags :access-tokens)
-                wrap-soft-auth))})
+        (handler request)))))
 
 (def authz
   {:name ::authz

backend/src/app/http/client.clj

@@ -9,8 +9,7 @@
   (:require
    [app.common.schema :as sm]
    [integrant.core :as ig]
-   [java-http-clj.core :as http]
-   [promesa.core :as p])
+   [java-http-clj.core :as http])
   (:import
    java.net.http.HttpClient))
 
@@ -29,14 +28,9 @@
 
 (defn send!
   ([client req] (send! client req {}))
-  ([client req {:keys [response-type sync?] :or {response-type :string sync? false}}]
+  ([client req {:keys [response-type] :or {response-type :string}}]
    (assert (client? client) "expected valid http client")
-   (if sync?
-     (http/send req {:client client :as response-type})
-     (try
-       (http/send-async req {:client client :as response-type})
-       (catch Throwable cause
-         (p/rejected cause))))))
+   (http/send req {:client client :as response-type})))
 
 (defn- resolve-client
   [params]
@@ -56,8 +50,8 @@
   ([cfg-or-client request]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request {:sync? true})))
+     (send! client request {})))
   ([cfg-or-client request options]
    (let [client  (resolve-client cfg-or-client)
          request (update request :uri str)]
-     (send! client request (merge {:sync? true} options)))))
+     (send! client request options))))

backend/src/app/http/errors.clj

@@ -13,6 +13,7 @@
    [app.config :as cf]
    [app.http :as-alias http]
    [app.http.access-token :as-alias actoken]
+   [app.http.auth :as-alias auth]
    [app.http.session :as-alias session]
    [app.util.inet :as inet]
    [clojure.spec.alpha :as s]
@@ -22,18 +23,16 @@
 (defn request->context
   "Extracts error report relevant context data from request."
   [request]
-  (let [claims (-> {}
-                   (into (::session/token-claims request))
-                   (into (::actoken/token-claims request)))]
-    {:request/path       (:path request)
-     :request/method     (:method request)
-     :request/params     (:params request)
-     :request/user-agent (yreq/get-header request "user-agent")
-     :request/ip-addr    (inet/parse-request request)
-     :request/profile-id (:uid claims)
-     :version/frontend   (or (yreq/get-header request "x-frontend-version") "unknown")
-     :version/backend    (:full cf/version)}))
-
+  (let [{:keys [claims] :as auth} (get request ::http/auth-data)]
+    (-> (cf/logging-context)
+        (assoc :request/path (:path request))
+        (assoc :request/method (:method request))
+        (assoc :request/params (:params request))
+        (assoc :request/user-agent (yreq/get-header request "user-agent"))
+        (assoc :request/ip-addr (inet/parse-request request))
+        (assoc :request/profile-id (get claims :uid))
+        (assoc :request/auth-data auth)
+        (assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
 
 (defmulti handle-error
   (fn [cause _ _]
@@ -61,7 +60,6 @@
        ::yres/body data}
 
       (binding [l/*context* (request->context request)]
-        (l/wrn :hint "restriction error" :cause err)
         {::yres/status 400
          ::yres/body data}))))
 

backend/src/app/http/management.clj

@@ -13,7 +13,7 @@
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
-   [app.http.access-token :refer [get-token]]
+   [app.http.middleware :as mw]
    [app.main :as-alias main]
    [app.rpc.commands.profile :as cmd.profile]
    [app.setup :as-alias setup]
@@ -32,20 +32,6 @@
   [_ params]
   (assert (db/pool? (::db/pool params)) "expect valid database pool"))
 
-(def ^:private auth
-  {:name ::auth
-   :compile
-   (fn [_ _]
-     (fn [handler shared-key]
-       (if shared-key
-         (fn [request]
-           (let [token (get-token request)]
-             (if (= token shared-key)
-               (handler request)
-               {::yres/status 403})))
-         (fn [_ _]
-           {::yres/status 403}))))})
-
 (def ^:private default-system
   {:name ::default-system
    :compile
@@ -65,7 +51,7 @@
 
 (defmethod ig/init-key ::routes
   [_ cfg]
-  ["" {:middleware [[auth (cf/get :management-api-shared-key)]
+  ["" {:middleware [[mw/shared-key-auth (cf/get :management-api-shared-key)]
                     [default-system cfg]
                     [transaction]]}
    ["/authenticate"

backend/src/app/http/middleware.clj

@@ -12,6 +12,7 @@
    [app.common.schema :as-alias sm]
    [app.common.transit :as t]
    [app.config :as cf]
+   [app.http :as-alias http]
    [app.http.errors :as errors]
    [app.util.pointer-map :as pmap]
    [cuerdas.core :as str]
@@ -240,3 +241,60 @@
              (if (contains? allowed method)
                (handler request)
                {::yres/status 405}))))))})
+
+
+(defn- wrap-auth
+  [handler decoders]
+  (let [token-re
+        #"(?i)^(Token|Bearer)\s+(.*)"
+
+        get-token-from-authorization
+        (fn [request]
+          (when-let [[_ token-type token] (some->> (yreq/get-header request "authorization")
+                                                   (re-matches token-re))]
+            (if (= "token" (str/lower token-type))
+              {:type :token
+               :token token}
+              {:type :bearer
+               :token token})))
+
+        get-token-from-cookie
+        (fn [request]
+          (let [cname (cf/get :auth-token-cookie-name)
+                token (some-> (yreq/get-cookie request cname) :value)]
+            (when-not (str/empty? token)
+              {:type :cookie
+               :token token})))
+
+        get-token
+        (some-fn get-token-from-cookie get-token-from-authorization)
+
+        process-request
+        (fn [request]
+          (if-let [{:keys [type token] :as auth} (get-token request)]
+            (if-let [decode-fn (get decoders type)]
+              (assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
+              (assoc request ::http/auth-data auth))
+            request))]
+
+    (fn [request]
+      (-> request process-request handler))))
+
+(def auth
+  {:name ::auth
+   :compile (constantly wrap-auth)})
+
+(defn- wrap-shared-key-auth
+  [handler shared-key]
+  (if shared-key
+    (fn [request]
+      (let [key (yreq/get-header request "x-shared-key")]
+        (if (= key shared-key)
+          (handler request)
+          {::yres/status 403})))
+    (fn [_ _]
+      {::yres/status 403})))
+
+(def shared-key-auth
+  {:name ::shared-key-auth
+   :compile (constantly wrap-shared-key-auth)})

backend/src/app/http/session.clj

@@ -11,28 +11,24 @@
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
+   [app.http :as-alias http]
+   [app.http.auth :as-alias http.auth]
    [app.http.session.tasks :as-alias tasks]
    [app.main :as-alias main]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
-   [cuerdas.core :as str]
    [integrant.core :as ig]
-   [yetti.request :as yreq]))
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; DEFAULTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; A default cookie name for storing the session.
-(def default-auth-token-cookie-name "auth-token")
-
-;; A cookie that we can use to check from other sites of the same
-;; domain if a user is authenticated.
-(def default-auth-data-cookie-name "auth-data")
-
 ;; Default value for cookie max-age
 (def default-cookie-max-age (ct/duration {:days 7}))
 
@@ -44,10 +40,10 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defprotocol ISessionManager
-  (read [_ key])
-  (write! [_ key data])
-  (update! [_ data])
-  (delete! [_ key]))
+  (read-session [_ id])
+  (create-session [_ params])
+  (update-session [_ session])
+  (delete-session [_ id]))
 
 (defn manager?
   [o]
@@ -62,71 +58,82 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private schema:params
-  [:map {:title "session-params"}
-   [:user-agent ::sm/text]
+  [:map {:title "SessionParams" :closed true}
    [:profile-id ::sm/uuid]
-   [:created-at ::ct/inst]])
+   [:user-agent {:optional true} ::sm/text]
+   [:sso-provider-id {:optional true} ::sm/uuid]
+   [:sso-session-id {:optional true} :string]])
 
 (def ^:private valid-params?
   (sm/validator schema:params))
 
-(defn- prepare-session-params
-  [params key]
-  (assert (string? key) "expected key to be a string")
-  (assert (not (str/blank? key)) "expected key to be not empty")
-  (assert (valid-params? params) "expected valid params")
-
-  {:user-agent (:user-agent params)
-   :profile-id (:profile-id params)
-   :created-at (:created-at params)
-   :updated-at (:created-at params)
-   :id key})
-
 (defn- database-manager
   [pool]
   (reify ISessionManager
-    (read [_ token]
-      (db/exec-one! pool (sql/select :http-session {:id token})))
+    (read-session [_ id]
+      (if (string? id)
+        ;; Backward compatibility
+        (let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
+          (-> session
+              (assoc :modified-at (:updated-at session))
+              (dissoc :updated-at)))
+        (db/exec-one! pool (sql/select :http-session-v2 {:id id}))))
 
-    (write! [_ key params]
-      (let [params (-> params
-                       (assoc :created-at (ct/now))
-                       (prepare-session-params key))]
-        (db/insert! pool :http-session params)
-        params))
+    (create-session [_ params]
+      (assert (valid-params? params) "expect valid session params")
 
-    (update! [_ params]
-      (let [updated-at (ct/now)]
-        (db/update! pool :http-session
-                    {:updated-at updated-at}
-                    {:id (:id params)})
-        (assoc params :updated-at updated-at)))
+      (let [now    (ct/now)
+            params (-> params
+                       (assoc :id (uuid/next))
+                       (assoc :created-at now)
+                       (assoc :modified-at now))]
+        (db/insert! pool :http-session-v2 params
+                    {::db/return-keys true})))
 
-    (delete! [_ token]
-      (db/delete! pool :http-session {:id token})
+    (update-session [_ session]
+      (let [modified-at (ct/now)]
+        (if (string? (:id session))
+          (let [params (-> session
+                           (assoc :id (uuid/next))
+                           (assoc :created-at modified-at)
+                           (assoc :modified-at modified-at))]
+            (db/insert! pool :http-session-v2 params))
+
+          (db/update! pool :http-session-v2
+                      {:modified-at modified-at}
+                      {:id (:id session)}))))
+
+    (delete-session [_ id]
+      (if (string? id)
+        (db/delete! pool :http-session {:id id} {::db/return-keys false})
+        (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
       nil)))
 
 (defn inmemory-manager
   []
   (let [cache (atom {})]
     (reify ISessionManager
-      (read [_ token]
-        (get @cache token))
+      (read-session [_ id]
+        (get @cache id))
 
-      (write! [_ key params]
-        (let [params (-> params
-                         (assoc :created-at (ct/now))
-                         (prepare-session-params key))]
-          (swap! cache assoc key params)
-          params))
+      (create-session [_ params]
+        (assert (valid-params? params) "expect valid session params")
 
-      (update! [_ params]
-        (let [updated-at (ct/now)]
-          (swap! cache update (:id params) assoc :updated-at updated-at)
-          (assoc params :updated-at updated-at)))
+        (let [now     (ct/now)
+              session (-> params
+                          (assoc :id (uuid/next))
+                          (assoc :created-at now)
+                          (assoc :modified-at now))]
+          (swap! cache assoc (:id session) session)
+          session))
 
-      (delete! [_ token]
-        (swap! cache dissoc token)
+      (update-session [_ session]
+        (let [modified-at (ct/now)]
+          (swap! cache update (:id session) assoc :modified-at modified-at)
+          (assoc session :modified-at modified-at)))
+
+      (delete-session [_ id]
+        (swap! cache dissoc id)
         nil))))
 
 (defmethod ig/assert-key ::manager
@@ -146,103 +153,114 @@
 ;; MANAGER IMPL
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare ^:private assign-auth-token-cookie)
-(declare ^:private clear-auth-token-cookie)
-(declare ^:private gen-token)
+(declare ^:private assign-session-cookie)
+(declare ^:private clear-session-cookie)
+
+(defn- assign-token
+  [cfg session]
+  (let [token (tokens/generate cfg
+                               {:iss "authentication"
+                                :aud "penpot"
+                                :sid (:id session)
+                                :iat (:modified-at session)
+                                :uid (:profile-id session)
+                                :sso-provider-id (:sso-provider-id session)
+                                :sso-session-id (:sso-session-id session)})]
+    (assoc session :token token)))
 
 (defn create-fn
-  [{:keys [::manager] :as cfg} profile-id]
+  [{:keys [::manager] :as cfg} {profile-id :id :as profile}
+   & {:keys [sso-provider-id sso-session-id]}]
+
   (assert (manager? manager) "expected valid session manager")
   (assert (uuid? profile-id) "expected valid uuid for profile-id")
 
   (fn [request response]
     (let [uagent  (yreq/get-header request "user-agent")
-          params  {:profile-id profile-id
-                   :user-agent uagent}
-          token   (gen-token cfg params)
-          session (write! manager token params)]
-      (l/trc :hint "create" :profile-id (str profile-id))
-      (-> response
-          (assign-auth-token-cookie session)))))
+          session (->> {:user-agent uagent
+                        :profile-id profile-id
+                        :sso-provider-id sso-provider-id
+                        :sso-session-id sso-session-id}
+                       (d/without-nils)
+                       (create-session manager)
+                       (assign-token cfg))]
+
+      (l/trc :hint "create" :id (str (:id session)) :profile-id (str profile-id))
+      (assign-session-cookie response session))))
 
 (defn delete-fn
   [{:keys [::manager]}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request response]
-    (let [cname   (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
-          cookie  (yreq/get-cookie request cname)]
-      (l/trc :hint "delete" :profile-id (:profile-id request))
-      (some->> (:value cookie) (delete! manager))
-      (-> response
-          (assoc :status 204)
-          (assoc :body nil)
-          (clear-auth-token-cookie)))))
+    (some->> (get request ::id) (delete-session manager))
+    (clear-session-cookie response)))
 
-(defn- gen-token
-  [cfg {:keys [profile-id created-at]}]
-  (tokens/generate cfg {:iss "authentication"
-                        :iat created-at
-                        :uid profile-id}))
-(defn- decode-token
+(defn decode-token
   [cfg token]
-  (when token
-    (tokens/verify cfg {:token token :iss "authentication"})))
+  (try
+    (tokens/verify cfg {:token token :iss "authentication"})
+    (catch Throwable cause
+      (l/trc :hint "exception on decoding token"
+             :token token
+             :cause cause))))
 
-(defn- get-token
+(defn get-session
   [request]
-  (let [cname  (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
-        cookie (some-> (yreq/get-cookie request cname) :value)]
-    (when-not (str/empty? cookie)
-      cookie)))
+  (get request ::session))
 
-(defn- get-session
-  [manager token]
-  (some->> token (read manager)))
+(defn invalidate-others
+  [cfg session]
+  (let [sql "delete from http_session_v2 where profile_id = ? and id != ?"]
+    (-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
+        (db/get-update-count))))
 
 (defn- renew-session?
-  [{:keys [updated-at] :as session}]
-  (and (ct/inst? updated-at)
-       (let [elapsed (ct/diff updated-at (ct/now))]
-         (neg? (compare default-renewal-max-age elapsed)))))
-
-(defn- wrap-soft-auth
-  [handler {:keys [::manager] :as cfg}]
-  (assert (manager? manager) "expected valid session manager")
-  (letfn [(handle-request [request]
-            (try
-              (let [token  (get-token request)
-                    claims (decode-token cfg token)]
-                (cond-> request
-                  (map? claims)
-                  (-> (assoc ::token-claims claims)
-                      (assoc ::token token))))
-              (catch Throwable cause
-                (l/trc :hint "exception on decoding malformed token" :cause cause)
-                request)))]
-
-    (fn [request]
-      (handler (handle-request request)))))
+  [{:keys [id modified-at] :as session}]
+  (or (string? id)
+      (and (ct/inst? modified-at)
+           (let [elapsed (ct/diff modified-at (ct/now))]
+             (neg? (compare default-renewal-max-age elapsed))))))
 
 (defn- wrap-authz
-  [handler {:keys [::manager]}]
+  [handler {:keys [::manager] :as cfg}]
   (assert (manager? manager) "expected valid session manager")
   (fn [request]
-    (let [session  (get-session manager (::token request))
-          request  (cond-> request
-                     (some? session)
-                     (assoc ::profile-id (:profile-id session)
-                            ::id (:id session)))
-          response (handler request)]
+    (let [{:keys [type token claims]} (get request ::http/auth-data)]
+      (cond
+        (= type :cookie)
+        (let [session (if-let [sid (:sid claims)]
+                        (read-session manager sid)
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
+                        (read-session manager token))
 
-      (if (renew-session? session)
-        (let [session (update! manager session)]
-          (-> response
-              (assign-auth-token-cookie session)))
-        response))))
+              request (cond-> request
+                        (some? session)
+                        (-> (assoc ::profile-id (:profile-id session))
+                            (assoc ::session session)))
 
-(def soft-auth
-  {:name ::soft-auth
-   :compile (constantly wrap-soft-auth)})
+              response (handler request)]
+
+          (if (renew-session? session)
+            (let [session (->> session
+                               (update-session manager)
+                               (assign-token cfg))]
+              (assign-session-cookie response session))
+            response))
+
+        (= type :bearer)
+        (let [session (if-let [sid (:sid claims)]
+                        (read-session manager sid)
+                        ;; BACKWARD COMPATIBILITY WITH OLD TOKENS
+                        (read-session manager token))
+
+              request (cond-> request
+                        (some? session)
+                        (-> (assoc ::profile-id (:profile-id session))
+                            (assoc ::session session)))]
+          (handler request))
+
+        :else
+        (handler request)))))
 
 (def authz
   {:name ::authz
@@ -250,16 +268,16 @@
 
 ;; --- IMPL
 
-(defn- assign-auth-token-cookie
-  [response {token :id updated-at :updated-at}]
+(defn- assign-session-cookie
+  [response {token :token modified-at :modified-at}]
   (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
-        created-at updated-at
+        created-at modified-at
         renewal    (ct/plus created-at default-renewal-max-age)
         expires    (ct/plus created-at max-age)
         secure?    (contains? cf/flags :secure-session-cookies)
         strict?    (contains? cf/flags :strict-session-cookies)
         cors?      (contains? cf/flags :cors)
-        name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+        name       (cf/get :auth-token-cookie-name)
         comment    (str "Renewal at: " (ct/format-inst renewal :rfc1123))
         cookie     {:path "/"
                     :http-only true
@@ -268,12 +286,12 @@
                     :comment comment
                     :same-site (if cors? :none (if strict? :strict :lax))
                     :secure secure?}]
-    (update response :cookies assoc name cookie)))
+    (update response ::yres/cookies assoc name cookie)))
 
-(defn- clear-auth-token-cookie
+(defn- clear-session-cookie
   [response]
-  (let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
-    (update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
+  (let [cname (cf/get :auth-token-cookie-name)]
+    (update response ::yres/cookies assoc cname {:path "/" :value "" :max-age 0})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TASK: SESSION GC

backend/src/app/loggers/audit.clj

@@ -25,7 +25,8 @@
    [app.util.inet :as inet]
    [app.util.services :as-alias sv]
    [app.worker :as wrk]
-   [cuerdas.core :as str]))
+   [cuerdas.core :as str]
+   [yetti.request :as yreq]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
@@ -90,6 +91,22 @@
      ::ip-addr (::rpc/ip-addr params)
      ::context (d/without-nils context)}))
 
+(defn get-external-session-id
+  [request]
+  (when-let [session-id (yreq/get-header request "x-external-session-id")]
+    (when-not (or (> (count session-id) 256)
+                  (= session-id "null")
+                  (str/blank? session-id))
+      session-id)))
+
+(defn- get-external-event-origin
+  [request]
+  (when-let [origin (yreq/get-header request "x-event-origin")]
+    (when-not (or (> (count origin) 256)
+                  (= origin "null")
+                  (str/blank? origin))
+      origin)))
+
 ;; --- SPECS
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -126,8 +143,6 @@
                          (::rpc/profile-id params)
                          uuid/zero)
 
-        session-id   (get params ::rpc/external-session-id)
-        event-origin (get params ::rpc/external-event-origin)
         props        (-> (or (::replace-props resultm)
                              (-> params
                                  (merge (::props resultm))
@@ -138,8 +153,10 @@
 
         token-id     (::actoken/id request)
         context      (-> (::context resultm)
-                         (assoc :external-session-id session-id)
-                         (assoc :external-event-origin event-origin)
+                         (assoc :external-session-id
+                                (get-external-session-id request))
+                         (assoc :external-event-origin
+                                (get-external-event-origin request))
                          (assoc :access-token-id (some-> token-id str))
                          (d/without-nils))
 

backend/src/app/main.clj

@@ -21,7 +21,7 @@
    [app.http.client :as-alias http.client]
    [app.http.debug :as-alias http.debug]
    [app.http.management :as mgmt]
-   [app.http.session :as-alias session]
+   [app.http.session :as session]
    [app.http.session.tasks :as-alias session.tasks]
    [app.http.websocket :as http.ws]
    [app.loggers.webhooks :as-alias webhooks]
@@ -31,7 +31,6 @@
    [app.redis :as-alias rds]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
-   [app.rpc.doc :as-alias rpc.doc]
    [app.setup :as-alias setup]
    [app.srepl :as-alias srepl]
    [app.storage :as-alias sto]
@@ -260,14 +259,17 @@
    ::oidc.providers/generic
    {::http.client/client (ig/ref ::http.client/client)}
 
+   ::oidc/providers
+   [(ig/ref ::oidc.providers/google)
+    (ig/ref ::oidc.providers/github)
+    (ig/ref ::oidc.providers/gitlab)
+    (ig/ref ::oidc.providers/generic)]
+
    ::oidc/routes
    {::http.client/client (ig/ref ::http.client/client)
     ::db/pool            (ig/ref ::db/pool)
     ::setup/props        (ig/ref ::setup/props)
-    ::oidc/providers     {:google (ig/ref ::oidc.providers/google)
-                          :github (ig/ref ::oidc.providers/github)
-                          :gitlab (ig/ref ::oidc.providers/gitlab)
-                          :oidc   (ig/ref ::oidc.providers/generic)}
+    ::oidc/providers     (ig/ref ::oidc/providers)
     ::session/manager    (ig/ref ::session/manager)
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
@@ -280,7 +282,6 @@
    {::session/manager    (ig/ref ::session/manager)
     ::db/pool            (ig/ref ::db/pool)
     ::rpc/routes         (ig/ref ::rpc/routes)
-    ::rpc.doc/routes     (ig/ref ::rpc.doc/routes)
     ::setup/props        (ig/ref ::setup/props)
     ::mtx/routes         (ig/ref ::mtx/routes)
     ::oidc/routes        (ig/ref ::oidc/routes)
@@ -300,6 +301,7 @@
    {::db/pool         (ig/ref ::db/pool)
     ::mtx/metrics     (ig/ref ::mtx/metrics)
     ::mbus/msgbus     (ig/ref ::mbus/msgbus)
+    ::setup/props     (ig/ref ::setup/props)
     ::session/manager (ig/ref ::session/manager)}
 
    :app.http.assets/routes
@@ -337,14 +339,26 @@
     ::email/blacklist    (ig/ref ::email/blacklist)
     ::email/whitelist    (ig/ref ::email/whitelist)}
 
-   :app.rpc.doc/routes
-   {:app.rpc/methods (ig/ref :app.rpc/methods)}
+   :app.rpc/management-methods
+   {::http.client/client (ig/ref ::http.client/client)
+    ::db/pool            (ig/ref ::db/pool)
+    ::rds/pool           (ig/ref ::rds/pool)
+    ::wrk/executor       (ig/ref ::wrk/netty-executor)
+    ::session/manager    (ig/ref ::session/manager)
+    ::sto/storage        (ig/ref ::sto/storage)
+    ::mtx/metrics        (ig/ref ::mtx/metrics)
+    ::mbus/msgbus        (ig/ref ::mbus/msgbus)
+    ::rds/client         (ig/ref ::rds/client)
+    ::setup/props        (ig/ref ::setup/props)}
 
    ::rpc/routes
-   {::rpc/methods     (ig/ref :app.rpc/methods)
-    ::db/pool         (ig/ref ::db/pool)
-    ::session/manager (ig/ref ::session/manager)
-    ::setup/props     (ig/ref ::setup/props)}
+   {::rpc/methods        (ig/ref :app.rpc/methods)
+    ::rpc/management-methods (ig/ref :app.rpc/management-methods)
+
+    ;; FIXME: revisit if db/pool is necessary here
+    ::db/pool                (ig/ref ::db/pool)
+    ::session/manager        (ig/ref ::session/manager)
+    ::setup/props            (ig/ref ::setup/props)}
 
    ::wrk/registry
    {::mtx/metrics (ig/ref ::mtx/metrics)

backend/src/app/media.clj

@@ -17,6 +17,7 @@
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as-alias db]
+   [app.http.client :as http]
    [app.storage :as-alias sto]
    [app.storage.tmp :as tmp]
    [buddy.core.bytes :as bb]
@@ -37,6 +38,9 @@
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
+(def default-max-file-size
+  (* 1024 1024 10)) ; 10 MiB
+
 (def schema:upload
   [:map {:title "Upload"}
    [:filename :string]
@@ -241,7 +245,7 @@
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (merge input info {:ts (ct/now)}))
+        (merge input info {:ts (ct/now) :size (fs/size path)}))
 
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
@@ -261,6 +265,7 @@
           (assoc input
                  :width  width
                  :height height
+                 :size (fs/size path)
                  :ts (ct/now)))))))
 
 (defmethod process-error org.im4java.core.InfoException
@@ -270,6 +275,54 @@
             :hint "invalid image"
             :cause error))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMAGE HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn download-image
+  "Download an image from the provided URI and return the media input object"
+  [{:keys [::http/client]} uri]
+  (letfn [(parse-and-validate [{:keys [headers] :as response}]
+            (let [size     (some-> (get headers "content-length") d/parse-integer)
+                  mtype    (get headers "content-type")
+                  format   (cm/mtype->format mtype)
+                  max-size (cf/get :media-max-file-size default-max-file-size)]
+
+              (when-not size
+                (ex/raise :type :validation
+                          :code :unknown-size
+                          :hint "seems like the url points to resource with unknown size"))
+
+              (when (> size max-size)
+                (ex/raise :type :validation
+                          :code :file-too-large
+                          :hint (str/ffmt "the file size % is greater than the maximum %"
+                                          size
+                                          default-max-file-size)))
+
+              (when (nil? format)
+                (ex/raise :type :validation
+                          :code :media-type-not-allowed
+                          :hint "seems like the url points to an invalid media object"))
+
+              {:size size :mtype mtype :format format}))]
+
+    (let [{:keys [body] :as response} (http/req! client
+                                                 {:method :get :uri uri}
+                                                 {:response-type :input-stream})
+          {:keys [size mtype]} (parse-and-validate response)
+          path    (tmp/tempfile :prefix "penpot.media.download.")
+          written (io/write* path body :size size)]
+
+      (when (not= written size)
+        (ex/raise :type :internal
+                  :code :mismatch-write-size
+                  :hint "unexpected state: unable to write to file"))
+
+      {;; :size size
+       :path path
+       :mtype mtype})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; FONTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

backend/src/app/migrations.clj

@@ -450,7 +450,13 @@
     :fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
 
    {:name "0141-add-file-data-table.sql"
-    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}])
+    :fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}
+
+   {:name "0142-add-sso-provider-table"
+    :fn (mg/resource "app/migrations/sql/0142-add-sso-provider-table.sql")}
+
+   {:name "0143-http-session-v2-table"
+    :fn (mg/resource "app/migrations/sql/0143-add-http-session-v2-table.sql")}])
 
 (defn apply-migrations!
   [pool name migrations]

backend/src/app/migrations/sql/0142-add-sso-provider-table.sql

@@ -0,0 +1,33 @@
+CREATE TABLE sso_provider (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  is_enabled boolean NOT NULL DEFAULT true,
+
+  type text NOT NULL CHECK (type IN ('oidc')),
+  domain text NOT NULL,
+
+  client_id text NOT NULL,
+  client_secret text NOT NULL,
+
+  base_uri text NOT NULL,
+  token_uri text NULL,
+  auth_uri text NULL,
+  user_uri text NULL,
+  jwks_uri text NULL,
+  logout_uri text NULL,
+
+  roles_attr text NULL,
+  email_attr text NULL,
+  name_attr text NULL,
+  user_info_source text NOT NULL DEFAULT 'token'
+     CHECK (user_info_source IN ('token', 'userinfo', 'auto')),
+
+  scopes text[] NULL,
+  roles text[] NULL
+);
+
+CREATE UNIQUE INDEX sso_provider__domain__idx
+    ON sso_provider(domain);

backend/src/app/migrations/sql/0143-add-http-session-v2-table.sql

@@ -0,0 +1,23 @@
+CREATE TABLE http_session_v2 (
+  id uuid PRIMARY KEY,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  modified_at timestamptz NOT NULL DEFAULT now(),
+
+  profile_id uuid REFERENCES profile(id) ON DELETE CASCADE,
+  user_agent text NULL,
+
+  sso_provider_id uuid NULL REFERENCES sso_provider(id) ON DELETE CASCADE,
+  sso_session_id text NULL
+);
+
+CREATE INDEX http_session_v2__profile_id__idx
+    ON http_session_v2(profile_id);
+
+CREATE INDEX http_session_v2__sso_provider_id__idx
+    ON http_session_v2(sso_provider_id)
+ WHERE sso_provider_id IS NOT NULL;
+
+CREATE INDEX http_session_v2__sso_session_id__idx
+    ON http_session_v2(sso_session_id)
+ WHERE sso_session_id IS NOT NULL;

backend/src/app/rpc.clj

@@ -13,11 +13,14 @@
    [app.common.schema :as sm]
    [app.common.spec :as us]
    [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
    [app.http.access-token :as actoken]
    [app.http.client :as-alias http.client]
+   [app.http.middleware :as mw]
+   [app.http.security :as sec]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as-alias main]
@@ -26,6 +29,7 @@
    [app.redis :as rds]
    [app.rpc.climit :as climit]
    [app.rpc.cond :as cond]
+   [app.rpc.doc :as doc]
    [app.rpc.helpers :as rph]
    [app.rpc.retry :as retry]
    [app.rpc.rlimit :as rlimit]
@@ -36,7 +40,6 @@
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [promesa.core :as p]
    [yetti.request :as yreq]
    [yetti.response :as yres]))
 
@@ -44,7 +47,7 @@
 
 (defn- default-handler
   [_]
-  (p/rejected (ex/error :type :not-found)))
+  (ex/raise :type :not-found))
 
 (defn- handle-response-transformation
   [response request mdata]
@@ -64,67 +67,58 @@
   (let [mdata    (meta result)
         response (if (fn? result)
                    (result request)
-                   (let [result (rph/unwrap result)]
-                     {::yres/status  (::http/status mdata 200)
-                      ::yres/headers (::http/headers mdata {})
+                   (let [result  (rph/unwrap result)
+                         status  (or (::http/status mdata)
+                                     (if (nil? result)
+                                       204
+                                       200))
+                         headers (cond-> (::http/headers mdata {})
+                                   (yres/stream-body? result)
+                                   (assoc "content-type" "application/octet-stream"))]
+                     {::yres/status  status
+                      ::yres/headers headers
                       ::yres/body    result}))]
+
     (-> response
         (handle-response-transformation request mdata)
         (handle-before-comple-hook mdata))))
 
-(defn get-external-session-id
-  [request]
-  (when-let [session-id (yreq/get-header request "x-external-session-id")]
-    (when-not (or (> (count session-id) 256)
-                  (= session-id "null")
-                  (str/blank? session-id))
-      session-id)))
-
-(defn- get-external-event-origin
-  [request]
-  (when-let [origin (yreq/get-header request "x-event-origin")]
-    (when-not (or (> (count origin) 256)
-                  (= origin "null")
-                  (str/blank? origin))
-      origin)))
-
-(defn- rpc-handler
+(defn- make-rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
-  [methods {:keys [params path-params method] :as request}]
-  (let [handler-name (:type path-params)
-        etag         (yreq/get-header request "if-none-match")
-        profile-id   (or (::session/profile-id request)
-                         (::actoken/profile-id request))
+  [methods]
+  (let [methods (update-vals methods peek)]
+    (fn [{:keys [params path-params method] :as request}]
+      (let [handler-name (:type path-params)
+            etag         (yreq/get-header request "if-none-match")
+            profile-id   (or (::session/profile-id request)
+                             (::actoken/profile-id request))
+            ip-addr      (inet/parse-request request)
 
-        ip-addr      (inet/parse-request request)
-        session-id   (get-external-session-id request)
-        event-origin (get-external-event-origin request)
+            data         (-> params
+                             (assoc ::handler-name handler-name)
+                             (assoc ::ip-addr ip-addr)
+                             (assoc ::request-at (ct/now))
+                             (assoc ::cond/key etag)
+                             (cond-> (uuid? profile-id)
+                               (assoc ::profile-id profile-id)))
 
-        data         (-> params
-                         (assoc ::handler-name handler-name)
-                         (assoc ::ip-addr ip-addr)
-                         (assoc ::request-at (ct/now))
-                         (assoc ::external-session-id session-id)
-                         (assoc ::external-event-origin event-origin)
-                         (assoc ::session/id (::session/id request))
-                         (assoc ::cond/key etag)
-                         (cond-> (uuid? profile-id)
-                           (assoc ::profile-id profile-id)))
+            data         (with-meta data
+                           {::http/request request})
 
-        data         (vary-meta data assoc ::http/request request)
-        handler-fn   (get methods (keyword handler-name) default-handler)]
+            handler-fn   (get methods (keyword handler-name) default-handler)]
 
-    (when (and (or (= method :get)
-                   (= method :head))
-               (not (str/starts-with? handler-name "get-")))
-      (ex/raise :type :restriction
-                :code :method-not-allowed
-                :hint "method not allowed for this request"))
+        (when (and (or (= method :get)
+                       (= method :head))
+                   (not (str/starts-with? handler-name "get-")))
+          (ex/raise :type :restriction
+                    :code :method-not-allowed
+                    :hint "method not allowed for this request"))
 
-    (binding [cond/*enabled* true]
-      (let [response (handler-fn data)]
-        (handle-response request response)))))
+        ;; FIXME: why we have this cond enabled here, we need to move it outside this handler
+        (binding [cond/*enabled* true]
+          (let [response (handler-fn data)]
+            (handle-response request response)))))))
 
 (defn- wrap-metrics
   "Wrap service method with metrics measurement."
@@ -201,7 +195,7 @@
                         ::sm/explain (explain params)))))))
     f))
 
-(defn- wrap-all
+(defn- wrap
   [cfg f mdata]
   (as-> f $
     (wrap-db-transaction cfg $ mdata)
@@ -215,17 +209,30 @@
     (wrap-params-validation cfg $ mdata)
     (wrap-authentication cfg $ mdata)))
 
-(defn- wrap
+(defn- wrap-management
   [cfg f mdata]
-  (l/trc :hint "register method" :name (::sv/name mdata))
-  (let [f (wrap-all cfg f mdata)]
-    (partial f cfg)))
+  (as-> f $
+    (wrap-db-transaction cfg $ mdata)
+    (retry/wrap-retry cfg $ mdata)
+    (climit/wrap cfg $ mdata)
+    (wrap-metrics cfg $ mdata)
+    (wrap-audit cfg $ mdata)
+    (wrap-spec-conform cfg $ mdata)
+    (wrap-params-validation cfg $ mdata)
+    (wrap-authentication cfg $ mdata)))
 
 (defn- process-method
-  [cfg [vfn mdata]]
-  [(keyword (::sv/name mdata)) [mdata (wrap cfg vfn mdata)]])
+  [cfg module wrap-fn [f mdata]]
+  (l/trc :hint "add method" :module module :name (::sv/name mdata))
+  (let [f (wrap-fn cfg f mdata)
+        k (keyword (::sv/name mdata))]
+    [k [mdata (partial f cfg)]]))
 
-(defn- resolve-command-methods
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; API METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-methods
   [cfg]
   (let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
     (->> (sv/scan-ns
@@ -254,7 +261,7 @@
           'app.rpc.commands.verify-token
           'app.rpc.commands.viewer
           'app.rpc.commands.webhooks)
-         (map (partial process-method cfg))
+         (map (partial process-method cfg "rpc" wrap))
          (into {}))))
 
 (def ^:private schema:methods-params
@@ -278,7 +285,49 @@
 (defmethod ig/init-key ::methods
   [_ cfg]
   (let [cfg (d/without-nils cfg)]
-    (resolve-command-methods cfg)))
+    (resolve-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MANAGEMENT METHODS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- resolve-management-methods
+  [cfg]
+  (let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
+    (->> (sv/scan-ns
+          'app.rpc.management.subscription)
+         (map (partial process-method cfg "management" wrap-management))
+         (into {}))))
+
+(def ^:private schema:management-methods-params
+  [:map {:title "management-methods-params"}
+   ::session/manager
+   ::http.client/client
+   ::db/pool
+   ::rds/pool
+   ::mbus/msgbus
+   ::sto/storage
+   ::mtx/metrics
+   ::setup/props])
+
+(defmethod ig/assert-key ::management-methods
+  [_ params]
+  (assert (sm/check schema:management-methods-params params)))
+
+(defmethod ig/init-key ::management-methods
+  [_ cfg]
+  (let [cfg (d/without-nils cfg)]
+    (resolve-management-methods cfg)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ROUTES
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- redirect
+  [href]
+  (fn [_]
+    {::yres/status 308
+     ::yres/headers {"location" (str href)}}))
 
 (def ^:private schema:methods
   [:map-of :keyword [:tuple :map ::sm/fn]])
@@ -293,11 +342,48 @@
   (assert (db/pool? (::db/pool params)) "expect valid database pool")
   (assert (some? (::setup/props params)))
   (assert (session/manager? (::session/manager params)) "expect valid session manager")
-  (assert (valid-methods? (::methods params)) "expect valid methods map"))
+  (assert (valid-methods? (::methods params)) "expect valid methods map")
+  (assert (valid-methods? (::management-methods params)) "expect valid methods map"))
 
 (defmethod ig/init-key ::routes
-  [_ {:keys [::methods] :as cfg}]
-  (let [methods (update-vals methods peek)]
-    [["/rpc" {:middleware [[session/authz cfg]
-                           [actoken/authz cfg]]}
-      ["/command/:type" {:handler (partial rpc-handler methods)}]]]))
+  [_ {:keys [::methods ::management-methods] :as cfg}]
+
+  (let [public-uri (cf/get :public-uri)]
+    ["/api"
+
+     ["/management"
+      ["/methods/:type"
+       {:middleware [[mw/shared-key-auth (cf/get :management-api-shared-key)]
+                     [session/authz cfg]]
+        :handler (make-rpc-handler management-methods)}]
+
+      (doc/routes :methods management-methods
+                  :label "management"
+                  :base-uri (u/join public-uri "/api/management")
+                  :description "MANAGEMENT API")]
+
+     ["/main"
+      ["/methods/:type"
+       {:middleware [[mw/cors]
+                     [sec/client-header-check]
+                     [session/authz cfg]
+                     [actoken/authz cfg]]
+        :handler (make-rpc-handler methods)}]
+
+      (doc/routes :methods methods
+                  :label "main"
+                  :base-uri (u/join public-uri "/api/main")
+                  :description "MAIN API")]
+
+     ;; BACKWARD COMPATIBILITY
+     ["/_doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
+     ["/openapi" {:handler (redirect (u/join public-uri "/api/main/doc/openapi"))}]
+     ["/openapi.join" {:handler (redirect (u/join public-uri "/api/main/doc/openapi.json"))}]
+
+     ["/rpc/command/:type"
+      {:middleware [[mw/cors]
+                    [sec/client-header-check]
+                    [session/authz cfg]
+                    [actoken/authz cfg]]
+       :handler (make-rpc-handler methods)}]]))

backend/src/app/rpc/commands/access_token.clj

@@ -28,6 +28,7 @@
         expires-at (some-> expiration (ct/in-future))
         created-at (ct/now)
         token      (tokens/generate cfg {:iss "access-token"
+                                         :uid profile-id
                                          :iat created-at
                                          :tid token-id})
 

backend/src/app/rpc/commands/auth.clj

@@ -7,21 +7,24 @@
 (ns app.rpc.commands.auth
   (:require
    [app.auth :as auth]
+   [app.auth.oidc :as oidc]
    [app.common.data :as d]
-   [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.features :as cfeat]
    [app.common.logging :as l]
    [app.common.schema :as sm]
    [app.common.time :as ct]
+   [app.common.uri :as u]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.email :as eml]
    [app.email.blacklist :as email.blacklist]
    [app.email.whitelist :as email.whitelist]
+   [app.http :as-alias http]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
+   [app.media :as media]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
    [app.rpc.commands.profile :as profile]
@@ -30,6 +33,7 @@
    [app.rpc.helpers :as rph]
    [app.setup :as-alias setup]
    [app.setup.welcome-file :refer [create-welcome-file]]
+   [app.storage :as sto]
    [app.tokens :as tokens]
    [app.util.services :as sv]
    [app.worker :as wrk]
@@ -109,7 +113,7 @@
                                (assoc profile :is-admin (let [admins (cf/get :admins)]
                                                           (contains? admins (:email profile)))))]
               (-> response
-                  (rph/with-transform (session/create-fn cfg (:id profile)))
+                  (rph/with-transform (session/create-fn cfg profile))
                   (rph/with-meta {::audit/props (audit/profile->props profile)
                                   ::audit/profile-id (:id profile)}))))]
 
@@ -145,7 +149,24 @@
   [cfg params]
   (if (= (:profile-id params)
          (::rpc/profile-id params))
-    (rph/with-transform {} (session/delete-fn cfg))
+    (let [{:keys [claims]}
+          (rph/get-auth-data params)
+
+          provider
+          (some->> (get claims :sso-provider-id)
+                   (oidc/get-provider cfg))
+
+          response
+          (if (and provider (:logout-uri provider))
+            (let [params {"logout_hint" (get claims :sso-session-id)
+                          "client_id" (get provider :client-id)
+                          "post_logout_redirect_uri" (str (cf/get :public-uri))}
+                  uri    (-> (u/uri (:logout-uri provider))
+                             (assoc :query (u/map->query-string params)))]
+              {:redirect-uri uri})
+            {})]
+
+      (rph/with-transform response (session/delete-fn cfg)))
     {}))
 
 ;; ---- COMMAND: Recover Profile
@@ -271,11 +292,29 @@
 
 ;; ---- COMMAND: Register Profile
 
-(defn create-profile!
+(defn import-profile-picture
+  [cfg uri]
+  (try
+    (let [storage (sto/resolve cfg)
+          input   (media/download-image cfg uri)
+          input   (media/run {:cmd :info :input input})
+          hash    (sto/calculate-hash (:path input))
+          content (-> (sto/content (:path input) (:size input))
+                      (sto/wrap-with-hash hash))
+          sobject (sto/put-object! storage {::sto/content content
+                                            ::sto/deduplicate? true
+                                            :bucket "profile"
+                                            :content-type (:mtype input)})]
+      (:id sobject))
+    (catch Throwable cause
+      (l/err :hint "unable to import profile picture"
+             :cause cause)
+      nil)))
+
+(defn create-profile
   "Create the profile entry on the database with limited set of input
   attrs (all the other attrs are filled with default values)."
-  [conn {:keys [email] :as params}]
-  (dm/assert! ::sm/email email)
+  [{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
   (let [id        (or (:id params) (uuid/next))
         props     (-> (audit/extract-utm-params params)
                       (merge (:props params))
@@ -283,8 +322,7 @@
                               :viewed-walkthrough? false
                               :nudge {:big 10 :small 1}
                               :v2-info-shown true
-                              :release-notes-viewed (:main cf/version)})
-                      (db/tjson))
+                              :release-notes-viewed (:main cf/version)}))
 
         password  (or (:password params) "!")
 
@@ -299,6 +337,12 @@
         theme     (:theme params nil)
         email     (str/lower email)
 
+        photo-id  (some->> (or (:oidc/picture props)
+                               (:google/picture props)
+                               (:github/picture props)
+                               (:gitlab/picture props))
+                           (import-profile-picture cfg))
+
         params    {:id id
                    :fullname (:fullname params)
                    :email email
@@ -306,27 +350,26 @@
                    :lang locale
                    :password password
                    :deleted-at (:deleted-at params)
-                   :props props
+                   :props (db/tjson props)
                    :theme theme
+                   :photo-id photo-id
                    :is-active is-active
                    :is-muted is-muted
                    :is-demo is-demo}]
+
     (try
       (-> (db/insert! conn :profile params)
           (profile/decode-row))
       (catch org.postgresql.util.PSQLException cause
-        (let [state (.getSQLState cause)]
-          (if (not= state "23505")
-            (throw cause)
+        (if (db/duplicate-key-error? cause)
+          (ex/raise :type :validation
+                    :code :email-already-exists
+                    :hint "email already exists"
+                    :cause cause)
+          (throw cause))))))
 
-            (do
-              (l/error :hint "not an error" :cause cause)
-              (ex/raise :type :validation
-                        :code :email-already-exists
-                        :hint "email already exists"
-                        :cause cause))))))))
 
-(defn create-profile-rels!
+(defn create-profile-rels
   [conn {:keys [id] :as profile}]
   (let [features (cfeat/get-enabled-features cf/flags)
         team     (teams/create-team conn
@@ -376,12 +419,13 @@
                      ;; to detect if the profile is already registered
                      (or (profile/get-profile-by-email conn (:email claims))
                          (let [is-active (or (boolean (:is-active claims))
+                                             (boolean (:email-verified claims))
                                              (not (contains? cf/flags :email-verification)))
                                params    (-> params
                                              (assoc :is-active is-active)
                                              (update :password auth/derive-password))
-                               profile   (->> (create-profile! conn params)
-                                              (create-profile-rels! conn))]
+                               profile   (->> (create-profile cfg params)
+                                              (create-profile-rels conn))]
                            (vary-meta profile assoc :created true))))
 
         created?   (-> profile meta :created true?)
@@ -419,10 +463,10 @@
       (and (some? invitation)
            (= (:email profile)
               (:member-email invitation)))
-      (let [claims (assoc invitation :member-id  (:id profile))
-            token  (tokens/generate cfg claims)]
+      (let [invitation (assoc invitation :member-id  (:id profile))
+            token      (tokens/generate cfg invitation)]
         (-> {:invitation-token token}
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-meta {::audit/replace-props props
                             ::audit/context {:action "accept-invitation"}
                             ::audit/profile-id (:id profile)})))
@@ -433,7 +477,7 @@
       created?
       (if (:is-active profile)
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile claims))
             (rph/with-defer create-welcome-file-when-needed)
             (rph/with-meta
               {::audit/replace-props props
@@ -562,4 +606,32 @@
   [cfg params]
   (db/tx-run! cfg request-profile-recovery params))
 
+;; --- COMMAND: get-sso-config
 
+(defn- extract-domain
+  "Extract the domain part from email"
+  [email]
+  (let [at (str/last-index-of email "@")]
+    (when (and (>= at 0)
+               (< at (dec (count email))))
+      (-> (subs email (inc at))
+          (str/trim)
+          (str/lower)))))
+
+(def ^:private schema:get-sso-provider
+  [:map {:title "get-sso-config"}
+   [:email ::sm/email]])
+
+(def ^:private schema:get-sso-provider-result
+  [:map {:title "SSOProvider"}
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::get-sso-provider
+  {::rpc/auth false
+   ::doc/added "2.12"
+   ::sm/params schema:get-sso-provider
+   ::sm/result schema:get-sso-provider-result}
+  [cfg {:keys [email]}]
+  (when-let [domain (extract-domain email)]
+    (when-let [config (db/get* cfg :sso-provider {:domain domain})]
+      (select-keys config [:id]))))

backend/src/app/rpc/commands/binfile.clj

@@ -25,10 +25,10 @@
    [app.rpc.commands.projects :as projects]
    [app.rpc.commands.teams :as teams]
    [app.rpc.doc :as-alias doc]
+   [app.rpc.helpers :as rph]
    [app.tasks.file-gc]
    [app.util.services :as sv]
-   [app.worker :as-alias wrk]
-   [yetti.response :as yres]))
+   [app.worker :as-alias wrk]))
 
 (set! *warn-on-reflection* true)
 
@@ -44,7 +44,7 @@
 
 (defn stream-export-v1
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (yres/stream-body
+  (rph/stream
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -59,7 +59,7 @@
 
 (defn stream-export-v3
   [cfg {:keys [file-id include-libraries embed-assets] :as params}]
-  (yres/stream-body
+  (rph/stream
    (fn [_ output-stream]
      (try
        (-> cfg
@@ -79,16 +79,11 @@
    ::sm/params schema:export-binfile}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
   (files/check-read-permissions! pool profile-id file-id)
-  (fn [_]
-    (let [version (or version 1)
-          body    (case (int version)
-                    1 (stream-export-v1 cfg params)
-                    2 (throw (ex-info "not-implemented" {}))
-                    3 (stream-export-v3 cfg params))]
-
-      {::yres/status 200
-       ::yres/headers {"content-type" "application/octet-stream"}
-       ::yres/body body})))
+  (let [version (or version 1)]
+    (case (int version)
+      1 (stream-export-v1 cfg params)
+      2 (throw (ex-info "not-implemented" {}))
+      3 (stream-export-v3 cfg params))))
 
 ;; --- Command: import-binfile
 

backend/src/app/rpc/commands/comments.clj

@@ -234,36 +234,39 @@
                  (files/check-comment-permissions! conn profile-id file-id share-id)
                  (get-comment-threads conn profile-id file-id))))
 
-(def ^:private sql:comment-threads
-  "SELECT DISTINCT ON (ct.id)
-          ct.*,
-          pf.fullname AS owner_fullname,
-          pf.email AS owner_email,
-          pf.photo_id AS owner_photo_id,
-          p.team_id AS team_id,
-          f.name AS file_name,
-          f.project_id AS project_id,
-          first_value(c.content) OVER w AS content,
-          (SELECT count(1)
-             FROM comment AS c
-            WHERE c.thread_id = ct.id) AS count_comments,
-          (SELECT count(1)
-             FROM comment AS c
-            WHERE c.thread_id = ct.id
-              AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
-     FROM comment_thread AS ct
-    INNER JOIN comment AS c ON (c.thread_id = ct.id)
-    INNER JOIN file AS f ON (f.id = ct.file_id)
-    INNER JOIN project AS p ON (p.id = f.project_id)
-     LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
-     LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
-    WHERE f.deleted_at IS NULL
-      AND p.deleted_at IS NULL
-   WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)")
+(defn- get-comment-threads-sql
+  [where]
+  (str/ffmt
+   "SELECT DISTINCT ON (ct.id)
+           ct.*,
+           pf.fullname AS owner_fullname,
+           pf.email AS owner_email,
+           pf.photo_id AS owner_photo_id,
+           p.team_id AS team_id,
+           f.name AS file_name,
+           f.project_id AS project_id,
+           first_value(c.content) OVER w AS content,
+           (SELECT count(1)
+              FROM comment AS c
+             WHERE c.thread_id = ct.id) AS count_comments,
+           (SELECT count(1)
+              FROM comment AS c
+             WHERE c.thread_id = ct.id
+               AND c.created_at >= coalesce(cts.modified_at, ct.created_at)) AS count_unread_comments
+      FROM comment_thread AS ct
+     INNER JOIN comment AS c ON (c.thread_id = ct.id)
+     INNER JOIN file AS f ON (f.id = ct.file_id)
+     INNER JOIN project AS p ON (p.id = f.project_id)
+      LEFT JOIN comment_thread_status AS cts ON (cts.thread_id = ct.id AND cts.profile_id = ?)
+      LEFT JOIN profile AS pf ON (ct.owner_id = pf.id)
+     WHERE f.deleted_at IS NULL
+       AND p.deleted_at IS NULL
+       %1
+    WINDOW w AS (PARTITION BY c.thread_id ORDER BY c.created_at ASC)"
+   where))
 
 (def ^:private sql:comment-threads-by-file-id
-  (str "WITH threads AS (" sql:comment-threads ")"
-       "SELECT * FROM threads WHERE file_id = ?"))
+  (get-comment-threads-sql "AND ct.file_id = ?"))
 
 (defn- get-comment-threads
   [conn profile-id file-id]
@@ -273,34 +276,29 @@
 ;; --- COMMAND: Get Unread Comment Threads
 
 (def ^:private sql:unread-all-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
-       "SELECT * FROM threads WHERE count_unread_comments > 0 AND team_id = ?"))
+  (str "WITH threads AS ("
+       (get-comment-threads-sql "AND p.team_id = ?")
+       ")"
+       "SELECT t.* FROM threads AS t
+         WHERE t.count_unread_comments > 0"))
 
-;; The partial configuration will retrieve only comments created by the user and
-;; threads that have a mention to the user.
 (def ^:private sql:unread-partial-comment-threads-by-team
-  (str "WITH threads AS (" sql:comment-threads ")"
-       "SELECT * FROM threads
-         WHERE count_unread_comments > 0
-           AND team_id = ?
-           AND (owner_id = ? OR ? = ANY(mentions))"))
+  (str "WITH threads AS ("
+       (get-comment-threads-sql "AND p.team_id = ? AND (ct.owner_id = ? OR ? = ANY(ct.mentions))")
+       ")"
+       "SELECT t.* FROM threads AS t
+         WHERE t.count_unread_comments > 0"))
 
 (defn- get-unread-comment-threads
   [cfg profile-id team-id]
-  (let [profile (-> (db/get cfg :profile {:id profile-id})
+  (let [profile (-> (db/get cfg :profile {:id profile-id} ::db/remove-deleted false)
                     (profile/decode-row))
-        notify  (or (-> profile :props :notifications :dashboard-comments) :all)]
-
-    (case notify
-      :all
-      (->> (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
-           (into [] xf-decode-row))
-
-      :partial
-      (->> (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
-           (into [] xf-decode-row))
-
-      [])))
+        notify  (or (-> profile :props :notifications :dashboard-comments) :all)
+        result  (case notify
+                  :all     (db/exec! cfg [sql:unread-all-comment-threads-by-team profile-id team-id])
+                  :partial (db/exec! cfg [sql:unread-partial-comment-threads-by-team profile-id team-id profile-id profile-id])
+                  [])]
+    (into [] xf-decode-row result)))
 
 (def ^:private
   schema:get-unread-comment-threads
@@ -323,16 +321,17 @@
    [:id ::sm/uuid]
    [:share-id {:optional true} [:maybe ::sm/uuid]]])
 
+(def ^:private sql:get-comment-thread
+  (get-comment-threads-sql "AND ct.file_id = ? AND ct.id = ?"))
+
 (sv/defmethod ::get-comment-thread
   {::doc/added "1.15"
    ::sm/params schema:get-comment-thread}
   [cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
   (db/run! cfg (fn [{:keys [::db/conn]}]
                  (files/check-comment-permissions! conn profile-id file-id share-id)
-                 (let [sql (str "WITH threads AS (" sql:comment-threads ")"
-                                "SELECT * FROM threads WHERE id = ? AND file_id = ?")]
-                   (-> (db/exec-one! conn [sql profile-id id file-id])
-                       (decode-row))))))
+                 (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
+                         (decode-row)))))
 
 ;; --- COMMAND: Retrieve Comments
 

backend/src/app/rpc/commands/demo.clj

@@ -45,12 +45,13 @@
         params   {:email email
                   :fullname fullname
                   :is-active true
+                  :is-demo true
                   :deleted-at (ct/in-future (cf/get-deletion-delay))
                   :password (derive-password password)
                   :props {}}
-        profile  (db/tx-run! cfg (fn [{:keys [::db/conn]}]
-                                   (->> (auth/create-profile! conn params)
-                                        (auth/create-profile-rels! conn))))]
+        profile  (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
+                                   (->> (auth/create-profile cfg params)
+                                        (auth/create-profile-rels conn))))]
     (with-meta {:email email
                 :password password}
       {::audit/profile-id (:id profile)})))

backend/src/app/rpc/commands/feedback.clj

@@ -7,6 +7,7 @@
 (ns app.rpc.commands.feedback
   "A general purpose feedback module."
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.schema :as sm]
    [app.config :as cf]
@@ -21,8 +22,11 @@
 
 (def ^:private schema:send-user-feedback
   [:map {:title "send-user-feedback"}
-   [:subject [:string {:max 400}]]
-   [:content [:string {:max 2500}]]])
+   [:subject [:string {:max 500}]]
+   [:content [:string {:max 2500}]]
+   [:type {:optional true} :string]
+   [:error-href {:optional true} [:string {:max 2500}]]
+   [:error-report {:optional true} :string]])
 
 (sv/defmethod ::send-user-feedback
   {::doc/added "1.18"
@@ -39,16 +43,26 @@
 
 (defn- send-user-feedback!
   [pool profile params]
-  (let [dest (or (cf/get :user-feedback-destination)
-                 ;; LEGACY
-                 (cf/get :feedback-destination))]
+  (let [destination
+        (or (cf/get :user-feedback-destination)
+            ;; LEGACY
+            (cf/get :feedback-destination))
+
+        attachments
+        (d/without-nils
+         {"error-report.txt" (:error-report params)})]
+
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/user-feedback
-                :from     dest
-                :to       dest
-                :profile  profile
+                :from     (cf/get :smtp-default-from)
+                :to       destination
                 :reply-to (:email profile)
                 :email    (:email profile)
-                :subject  (:subject params)
-                :content  (:content params)})
+                :attachments attachments
+
+                :feedback-subject (:subject params)
+                :feedback-type (:type params "not-specified")
+                :feedback-content (:content params)
+                :feedback-error-href (:error-href params)
+                :profile profile})
     nil))

backend/src/app/rpc/commands/files.clj

@@ -26,6 +26,7 @@
    [app.db.sql :as-alias sql]
    [app.features.fdata :as feat.fdata]
    [app.features.logical-deletion :as ldel]
+   [app.http.sse :as sse]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.msgbus :as mbus]
@@ -38,6 +39,7 @@
    [app.rpc.helpers :as rph]
    [app.rpc.permissions :as perms]
    [app.util.blob :as blob]
+   [app.util.events :as events]
    [app.util.pointer-map :as pmap]
    [app.util.services :as sv]
    [app.worker :as wrk]
@@ -353,9 +355,8 @@
    ::sm/params schema:get-project-files
    ::sm/result schema:files}
   [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id]}]
-  (dm/with-open [conn (db/open pool)]
-    (projects/check-read-permissions! conn profile-id project-id)
-    (get-project-files conn project-id)))
+  (projects/check-read-permissions! pool profile-id project-id)
+  (get-project-files pool project-id))
 
 ;; --- COMMAND QUERY: has-file-libraries
 
@@ -424,7 +425,6 @@
 
 ;; --- QUERY COMMAND: get-page
 
-
 (defn- prune-objects
   "Given the page data and the object-id returns the page data with all
   other not needed objects removed from the `:objects` data
@@ -765,6 +765,54 @@
     (teams/check-read-permissions! conn profile-id team-id)
     (get-team-recent-files conn team-id)))
 
+
+;; --- COMMAND QUERY: get-team-deleted-files
+
+(def sql:team-deleted-files
+  "WITH deleted_files AS (
+     SELECT f.id,
+            f.revn,
+            f.vern,
+            f.project_id,
+            f.created_at,
+            f.modified_at,
+            f.name,
+            f.is_shared,
+            f.deleted_at AS will_be_deleted_at,
+            ft.media_id AS thumbnail_id,
+            row_number() OVER w AS row_num,
+            p.team_id
+       FROM file AS f
+      INNER JOIN project AS p ON (p.id = f.project_id)
+       LEFT JOIN file_thumbnail AS ft on (ft.file_id = f.id
+                                          AND ft.revn = f.revn
+                                          AND ft.deleted_at is null)
+      WHERE p.team_id = ?
+        AND (p.deleted_at > ?::timestamptz OR
+             f.deleted_at > ?::timestamptz)
+     WINDOW w AS (PARTITION BY f.project_id
+                      ORDER BY f.modified_at DESC)
+      ORDER BY f.modified_at DESC
+   )
+   SELECT * FROM deleted_files")
+
+(defn get-team-deleted-files
+  [conn team-id]
+  (let [now (ct/now)]
+    (db/exec! conn [sql:team-deleted-files team-id now now])))
+
+(def ^:private schema:get-team-deleted-files
+  [:map {:title "get-team-deleted-files"}
+   [:team-id ::sm/uuid]])
+
+(sv/defmethod ::get-team-deleted-files
+  {::doc/added "2.12"
+   ::sm/params schema:get-team-deleted-files}
+  [cfg {:keys [::rpc/profile-id team-id]}]
+  (db/run! cfg (fn [{:keys [::db/conn]}]
+                 (teams/check-read-permissions! conn profile-id team-id)
+                 (get-team-deleted-files conn team-id))))
+
 ;; --- COMMAND QUERY: get-file-info
 
 
@@ -1113,3 +1161,118 @@
   (check-edition-permissions! conn profile-id file-id)
   (->  (ignore-sync conn params)
        (update :features db/decode-pgarray #{})))
+
+;; --- MUTATION COMMAND: delete-files-immediatelly
+
+(def ^:private sql:delete-team-files
+  "UPDATE file AS uf SET deleted_at = ?::timestamptz
+     FROM (
+        SELECT f.id
+          FROM file AS f
+          JOIN project AS p ON (p.id = f.project_id)
+          JOIN team AS t ON (t.id = p.team_id)
+         WHERE t.deleted_at IS NULL
+           AND t.id = ?
+           AND f.id = ANY(?::uuid[])
+     ) AS subquery
+    WHERE uf.id = subquery.id
+   RETURNING uf.id, uf.deleted_at;")
+
+(def ^:private schema:permanently-delete-team-files
+  [:map {:title "permanently-delete-team-files"}
+   [:team-id ::sm/uuid]
+   [:ids [::sm/set ::sm/uuid]]])
+
+(sv/defmethod ::permanently-delete-team-files
+  "Mark the specified files to be deleted immediatelly on the
+  specified team. The team-id on params will be used to filter and
+  check writable permissons on team."
+
+  {::doc/added "2.12"
+   ::sm/params schema:permanently-delete-team-files
+   ::db/transaction true}
+
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at team-id ids]}]
+  (teams/check-edition-permissions! conn profile-id team-id)
+
+  (reduce (fn [acc {:keys [id deleted-at]}]
+            (wrk/submit! {::db/conn conn
+                          ::wrk/task :delete-object
+                          ::wrk/params {:object :file
+                                        :deleted-at deleted-at
+                                        :id id}})
+            (conj acc id))
+          #{}
+          (db/plan conn [sql:delete-team-files request-at team-id
+                         (db/create-array conn "uuid" ids)])))
+
+;; --- MUTATION COMMAND: restore-files-immediatelly
+
+(def ^:private sql:resolve-editable-files
+  "SELECT f.id
+     FROM file AS f
+     JOIN project AS p ON (p.id = f.project_id)
+     JOIN team AS t ON (t.id = p.team_id)
+    WHERE t.deleted_at IS NULL
+      AND t.id = ?
+      AND f.id = ANY(?::uuid[])")
+
+(defn- restore-file
+  [conn file-id]
+  (db/update! conn :file
+              {:deleted-at nil
+               :has-media-trimmed false}
+              {:id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-media-object
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-change
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-data
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-thumbnail
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false})
+
+  (db/update! conn :file-tagged-object-thumbnail
+              {:deleted-at nil}
+              {:file-id file-id}
+              {::db/return-keys false}))
+
+(defn- restore-deleted-team-files
+  [{:keys [::db/conn]} {:keys [::rpc/profile-id team-id ids]}]
+  (teams/check-edition-permissions! conn profile-id team-id)
+
+  (reduce (fn [affected {:keys [id]}]
+            (let [index (inc (count affected))]
+              (events/tap :progress {:file-id id :index index :total (count ids)})
+              (restore-file conn id)
+              (conj affected id)))
+          #{}
+          (db/plan conn [sql:resolve-editable-files team-id
+                         (db/create-array conn "uuid" ids)])))
+
+(def ^:private schema:restore-deleted-team-files
+  [:map {:title "restore-deleted-team-files"}
+   [:team-id ::sm/uuid]
+   [:ids [::sm/set ::sm/uuid]]])
+
+(sv/defmethod ::restore-deleted-team-files
+  "Removes the deletion mark from the specified files (and respective projects)."
+
+  {::doc/added "2.12"
+   ::sse/stream? true
+   ::sm/params schema:restore-deleted-team-files}
+  [cfg params]
+  (sse/response #(db/tx-run! cfg restore-deleted-team-files params)))

backend/src/app/rpc/commands/ldap.clj

@@ -66,12 +66,12 @@
                             :member-email (:email profile))
               token  (tokens/generate cfg claims)]
           (-> {:invitation-token token}
-              (rph/with-transform (session/create-fn cfg (:id profile)))
+              (rph/with-transform (session/create-fn cfg profile))
               (rph/with-meta {::audit/props (:props profile)
                               ::audit/profile-id (:id profile)})))
 
         (-> (profile/strip-private-attrs profile)
-            (rph/with-transform (session/create-fn cfg (:id profile)))
+            (rph/with-transform (session/create-fn cfg profile))
             (rph/with-meta {::audit/props (:props profile)
                             ::audit/profile-id (:id profile)}))))))
 
@@ -83,6 +83,6 @@
                              (profile/clean-email)
                              (profile/get-profile-by-email conn))
                     (->> (assoc info :is-active true :is-demo false)
-                         (auth/create-profile! conn)
-                         (auth/create-profile-rels! conn)
+                         (auth/create-profile cfg)
+                         (auth/create-profile-rels conn)
                          (profile/strip-private-attrs))))))

backend/src/app/rpc/commands/media.clj

@@ -7,14 +7,10 @@
 (ns app.rpc.commands.media
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
-   [app.http.client :as http]
    [app.loggers.audit :as-alias audit]
    [app.media :as media]
    [app.rpc :as-alias rpc]
@@ -22,13 +18,7 @@
    [app.rpc.commands.files :as files]
    [app.rpc.doc :as-alias doc]
    [app.storage :as sto]
-   [app.storage.tmp :as tmp]
-   [app.util.services :as sv]
-   [cuerdas.core :as str]
-   [datoteka.io :as io]))
-
-(def default-max-file-size
-  (* 1024 1024 10)) ; 10 MiB
+   [app.util.services :as sv]))
 
 (def thumbnail-options
   {:width 100
@@ -197,56 +187,12 @@
 
     mobj))
 
-(defn download-image
-  [{:keys [::http/client]} uri]
-  (letfn [(parse-and-validate [{:keys [headers] :as response}]
-            (let [size     (some-> (get headers "content-length") d/parse-integer)
-                  mtype    (get headers "content-type")
-                  format   (cm/mtype->format mtype)
-                  max-size (cf/get :media-max-file-size default-max-file-size)]
-
-              (when-not size
-                (ex/raise :type :validation
-                          :code :unknown-size
-                          :hint "seems like the url points to resource with unknown size"))
-
-              (when (> size max-size)
-                (ex/raise :type :validation
-                          :code :file-too-large
-                          :hint (str/ffmt "the file size % is greater than the maximum %"
-                                          size
-                                          default-max-file-size)))
-
-              (when (nil? format)
-                (ex/raise :type :validation
-                          :code :media-type-not-allowed
-                          :hint "seems like the url points to an invalid media object"))
-
-              {:size size :mtype mtype :format format}))]
-
-    (let [{:keys [body] :as response} (http/req! client
-                                                 {:method :get :uri uri}
-                                                 {:response-type :input-stream :sync? true})
-          {:keys [size mtype]} (parse-and-validate response)
-          path    (tmp/tempfile :prefix "penpot.media.download.")
-          written (io/write* path body :size size)]
-
-      (when (not= written size)
-        (ex/raise :type :internal
-                  :code :mismatch-write-size
-                  :hint "unexpected state: unable to write to file"))
-
-      {:filename "tempfile"
-       :size size
-       :path path
-       :mtype mtype})))
-
 (defn- create-file-media-object-from-url
   [cfg {:keys [url name] :as params}]
-  (let [content (download-image cfg url)
+  (let [content (media/download-image cfg url)
         params  (-> params
                     (assoc :content content)
-                    (assoc :name (or name (:filename content))))]
+                    (assoc :name (d/nilv name "unknown")))]
 
     ;; NOTE: we use the climit here in a dynamic invocation because we
     ;; don't want saturate the process-image limit with IO (download

backend/src/app/rpc/commands/profile.clj

@@ -107,7 +107,9 @@
 (defn get-profile
   "Get profile by id. Throws not-found exception if no profile found."
   [conn id & {:as opts}]
-  (-> (db/get-by-id conn :profile id opts)
+  ;; NOTE: We need to set ::db/remove-deleted to false because demo profiles
+  ;; are created with a set deleted-at value
+  (-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false))
       (decode-row)))
 
 ;; --- MUTATION: Update Profile (own)
@@ -152,7 +154,6 @@
 
 (declare validate-password!)
 (declare update-profile-password!)
-(declare invalidate-profile-session!)
 
 (def ^:private
   schema:update-profile-password
@@ -167,8 +168,7 @@
    ::climit/id :auth/global
    ::db/transaction true}
   [cfg {:keys [::rpc/profile-id password] :as params}]
-  (let [profile    (validate-password! cfg (assoc params :profile-id profile-id))
-        session-id (::session/id params)]
+  (let [profile (validate-password! cfg (assoc params :profile-id profile-id))]
 
     (when (= (:email profile) (str/lower (:password params)))
       (ex/raise :type :validation
@@ -176,14 +176,12 @@
                 :hint "you can't use your email as password"))
 
     (update-profile-password! cfg (assoc profile :password password))
-    (invalidate-profile-session! cfg profile-id session-id)
-    nil))
 
-(defn- invalidate-profile-session!
-  "Removes all sessions except the current one."
-  [{:keys [::db/conn]} profile-id session-id]
-  (let [sql "delete from http_session where profile_id = ? and id != ?"]
-    (:next.jdbc/update-count (db/exec-one! conn [sql profile-id session-id]))))
+    (->> (rph/get-request params)
+         (session/get-session)
+         (session/invalidate-others cfg))
+
+    nil))
 
 (defn- validate-password!
   [{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
@@ -282,9 +280,9 @@
                          :file-path (str (:path file))
                          :file-mtype (:mtype file)}}))))
 
-(defn- generate-thumbnail!
-  [_ file]
-  (let [input   (media/run {:cmd :info :input file})
+(defn- generate-thumbnail
+  [_ input]
+  (let [input   (media/run {:cmd :info :input input})
         thumb   (media/run {:cmd :profile-thumbnail
                             :format :jpeg
                             :quality 85
@@ -305,7 +303,7 @@
                    (assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
                                        [:process-image/global]])
                    (assoc ::climit/label "upload-photo")
-                   (climit/invoke! generate-thumbnail! file))]
+                   (climit/invoke! generate-thumbnail file))]
     (sto/put-object! storage params)))
 
 ;; --- MUTATION: Request Email Change
@@ -473,13 +471,16 @@
           p.fullname AS name,
           p.email AS email
      FROM team_profile_rel AS tpr1
+     JOIN team as t
+       ON tpr1.team_id = t.id
      JOIN team_profile_rel AS tpr2
        ON (tpr1.team_id = tpr2.team_id)
      JOIN profile AS p
        ON (tpr2.profile_id = p.id)
     WHERE tpr1.profile_id = ?
       AND tpr1.is_owner IS true
-      AND tpr2.can_edit IS true")
+      AND tpr2.can_edit IS true
+      AND t.deleted_at IS NULL")
 
 (sv/defmethod ::get-subscription-usage
   {::doc/added "2.9"}

backend/src/app/rpc/commands/projects.clj

@@ -70,7 +70,27 @@
 
 ;; --- QUERY: Get projects
 
-(declare get-projects)
+(def ^:private sql:projects
+  "SELECT p.*,
+          coalesce(tpp.is_pinned, false) as is_pinned,
+          (SELECT count(*) FROM file AS f
+            WHERE f.project_id = p.id
+              AND f.deleted_at is null) AS count,
+          (SELECT count(*) FROM file AS f
+            WHERE f.project_id = p.id) AS total_count
+     FROM project AS p
+    INNER JOIN team AS t ON (t.id = p.team_id)
+     LEFT JOIN team_project_profile_rel AS tpp
+            ON (tpp.project_id = p.id AND
+                tpp.team_id = p.team_id AND
+                tpp.profile_id = ?)
+    WHERE p.team_id = ?
+      AND t.deleted_at is null
+    ORDER BY p.modified_at DESC")
+
+(defn get-projects
+  [conn profile-id team-id]
+  (db/exec! conn [sql:projects profile-id team-id]))
 
 (def ^:private schema:get-projects
   [:map {:title "get-projects"}
@@ -78,32 +98,11 @@
 
 (sv/defmethod ::get-projects
   {::doc/added "1.18"
+   ::doc/changes [["2.12" "This endpoint now return deleted but recoverable projects"]]
    ::sm/params schema:get-projects}
-  [{:keys [::db/pool]} {:keys [::rpc/profile-id team-id]}]
-  (dm/with-open [conn (db/open pool)]
-    (teams/check-read-permissions! conn profile-id team-id)
-    (get-projects conn profile-id team-id)))
-
-(def sql:projects
-  "select p.*,
-          coalesce(tpp.is_pinned, false) as is_pinned,
-          (select count(*) from file as f
-            where f.project_id = p.id
-              and deleted_at is null) as count
-     from project as p
-    inner join team as t on (t.id = p.team_id)
-     left join team_project_profile_rel as tpp
-            on (tpp.project_id = p.id and
-                tpp.team_id = p.team_id and
-                tpp.profile_id = ?)
-    where p.team_id = ?
-      and p.deleted_at is null
-      and t.deleted_at is null
-    order by p.modified_at desc")
-
-(defn get-projects
-  [conn profile-id team-id]
-  (db/exec! conn [sql:projects profile-id team-id]))
+  [cfg {:keys [::rpc/profile-id team-id]}]
+  (teams/check-read-permissions! cfg profile-id team-id)
+  (get-projects cfg profile-id team-id))
 
 ;; --- QUERY: Get all projects
 

backend/src/app/rpc/commands/teams.clj

@@ -37,14 +37,14 @@
 ;; --- Helpers & Specs
 
 (def ^:private sql:team-permissions
-  "select tpr.is_owner,
+  "SELECT tpr.is_owner,
           tpr.is_admin,
           tpr.can_edit
-     from team_profile_rel as tpr
-     join team as t on (t.id = tpr.team_id)
-    where tpr.profile_id = ?
-      and tpr.team_id = ?
-      and t.deleted_at is null")
+     FROM team_profile_rel AS tpr
+     JOIN team AS t ON (t.id = tpr.team_id)
+    WHERE tpr.profile_id = ?
+      AND tpr.team_id = ?
+      AND t.deleted_at IS NULL")
 
 (defn get-permissions
   [conn profile-id team-id]

backend/src/app/rpc/commands/verify_token.clj

@@ -73,7 +73,7 @@
                   {:id (:id profile)}))
 
     (-> claims
-        (rph/with-transform (session/create-fn cfg profile-id))
+        (rph/with-transform (session/create-fn cfg profile))
         (rph/with-meta {::audit/name "verify-profile-email"
                         ::audit/props (audit/profile->props profile)
                         ::audit/profile-id (:id profile)}))))

backend/src/app/rpc/doc.clj

@@ -16,6 +16,7 @@
    [app.common.schema.desc-native :as smdn]
    [app.common.schema.openapi :as oapi]
    [app.common.schema.registry :as sr]
+   [app.common.uri :as u]
    [app.config :as cf]
    [app.http.sse :as-alias sse]
    [app.loggers.webhooks :as-alias webhooks]
@@ -25,7 +26,6 @@
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [integrant.core :as ig]
    [pretty-spec.core :as ps]
    [yetti.response :as-alias yres]))
 
@@ -33,8 +33,8 @@
 ;; DOC (human readable)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- prepare-doc-context
-  [methods]
+(defn- context
+  [{:keys [methods entrypoint label openapi]}]
   (letfn [(fmt-spec [mdata]
             (when-let [spec (ex/ignoring (s/spec (::sv/spec mdata)))]
               (with-out-str
@@ -62,8 +62,10 @@
              :added (::added mdata)
              :changes (some->> (::changes mdata) (partition-all 2) (map vec))
              :spec (fmt-spec mdata)
-             :entrypoint (str (cf/get :public-uri) "/api/rpc/command/" (::sv/name mdata))
-
+             :entrypoint (-> entrypoint
+                             (u/ensure-path-slash)
+                             (u/join (::sv/name mdata))
+                             (str))
              :params-schema-js   (fmt-schema :js mdata ::sm/params)
              :result-schema-js   (fmt-schema :js mdata ::sm/result)
              :webhook-schema-js  (fmt-schema :js mdata ::sm/webhook)
@@ -72,6 +74,9 @@
              :webhook-schema-clj (fmt-schema :clj mdata ::sm/webhook)})]
 
     {:version (:main cf/version)
+     :label label
+     :entrypoint (str entrypoint)
+     :openapi (str openapi)
      :methods
      (->> methods
           (map val)
@@ -80,17 +85,19 @@
           (map get-context)
           (sort-by (juxt :module :name)))}))
 
-(defn- doc-handler
-  [context]
+(defn- handler
+  [& {:keys [template] :as options}]
   (if (contains? cf/flags :backend-api-doc)
-    (fn [request]
-      (let [params  (:query-params request)
-            pstyle  (:type params "js")
-            context (assoc @context :param-style pstyle)]
+    (let [context  (delay (context options))
+          template (or template "app/templates/api-doc.tmpl")]
+      (fn [request]
+        (let [params  (:query-params request)
+              pstyle  (:type params "js")
+              context (assoc @context :param-style pstyle)]
 
-        {::yres/status 200
-         ::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
-                         (tmpl/render context))}))
+          {::yres/status 200
+           ::yres/body (-> (io/resource template)
+                           (tmpl/render context))})))
     (fn [_]
       {::yres/status 404})))
 
@@ -98,8 +105,8 @@
 ;; OPENAPI / SWAGGER (v3.1)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn prepare-openapi-context
-  [methods]
+(defn- openapi-context
+  [{:keys [methods entrypoint description]}]
   (let [definitions (atom {})
         options {:registry sr/default-registry
                  ::oapi/definitions-path "#/components/schemas/"
@@ -112,7 +119,9 @@
         (fn [tsx schema]
           (let [schema  (sm/schema schema)
                 example (sm/generate schema)
-                example (sm/encode schema example output-transformer)]
+                example (sm/encode schema example output-transformer)
+                example (json/encode example :key-fn json/write-camel-key)]
+
             {:default
              {:description "A default response"
               :content
@@ -123,7 +132,9 @@
         gen-params-doc
         (fn [tsx schema]
           (let [example (sm/generate schema)
-                example (sm/encode schema example output-transformer)]
+                example (sm/encode schema example output-transformer)
+                example (json/encode example :key-fn json/write-camel-key)]
+
             {:required true
              :content
              {"application/json"
@@ -158,34 +169,35 @@
                (map gen-method-doc)
                (sort-by (juxt :module :name))
                (map (fn [doc]
-                      [(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
+                      [(:name doc) (:repr doc)]))
                (into {})))]
 
     {:openapi "3.0.0"
      :info {:version (:main cf/version)}
-     :servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
-                ;; :description "penpot backend"
-                }]
+     :servers [{:url (str entrypoint)
+                :description (or description "")}]
      :paths paths
      :components {:schemas @definitions}}))
 
-(defn openapi-json-handler
-  [context]
+(defn- openapi-json-handler
+  [& {:as options}]
   (if (contains? cf/flags :backend-openapi-doc)
-    (fn [_]
-      {::yres/status 200
-       ::yres/headers {"content-type" "application/json; charset=utf-8"}
-       ::yres/body (json/encode @context)})
+    (let [context (delay (openapi-context options))]
+      (fn [_]
+        {::yres/status 200
+         ::yres/headers {"content-type" "application/json; charset=utf-8"}
+         ::yres/body (json/encode @context)}))
     (fn [_]
       {::yres/status 404})))
 
-(defn openapi-handler
-  []
+(defn- openapi-handler
+  [& {:keys [uri label]}]
   (if (contains? cf/flags :backend-openapi-doc)
     (fn [_]
       (let [swagger-js (slurp (io/resource "app/assets/swagger-ui-4.18.3.js"))
             swagger-cs (slurp (io/resource "app/assets/swagger-ui-4.18.3.css"))
-            context    {:public-uri (cf/get :public-uri)
+            context    {:uri (str uri)
+                        :label label
                         :swagger-js swagger-js
                         :swagger-css swagger-cs}]
         {::yres/status 200
@@ -196,27 +208,43 @@
       {::yres/status 404})))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; MODULE INIT
+;; ROUTES HELPER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defmethod ig/assert-key ::routes
-  [_ params]
-  (assert (sm/valid? ::rpc/methods (::rpc/methods params)) "expected valid methods"))
+(defn routes
+  [& {:keys [label base-uri description methods]}]
+  (let [entrypoint
+        (-> base-uri
+            (u/ensure-path-slash)
+            (u/join "methods"))
 
-(defmethod ig/init-key ::routes
-  [_ {:keys [::rpc/methods] :as cfg}]
-  [(let [context (delay (prepare-doc-context methods))]
-     [["/_doc"
-       {:handler (doc-handler context)
-        :allowed-methods #{:get}}]
-      ["/doc"
-       {:handler (doc-handler context)
-        :allowed-methods #{:get}}]])
+        openapi
+        (-> base-uri
+            (u/ensure-path-slash)
+            (u/join "doc/openapi"))
 
-   (let [context (delay (prepare-openapi-context methods))]
-     [["/openapi"
-       {:handler (openapi-handler)
-        :allowed-methods #{:get}}]
-      ["/openapi.json"
-       {:handler (openapi-json-handler context)
-        :allowed-methods #{:get}}]])])
+        template
+        (case label
+          "management" "app/templates/management-api-doc.tmpl"
+          "main"       "app/templates/main-api-doc.tmpl")]
+
+    ["/doc"
+     ["" {:handler (handler :methods methods
+                            :label label
+                            :entrypoint entrypoint
+                            :openapi openapi
+                            :template template)
+          :allowed-methods #{:get}}]
+
+     ["/openapi"
+      {:handler (openapi-handler
+                 :uri (u/join openapi "openapi.json")
+                 :label label)
+       :allowed-methods #{:get}}]
+
+     ["/openapi.json"
+      {:handler (openapi-json-handler {:entrypoint entrypoint
+                                       :description description
+                                       :methods methods})
+
+       :allowed-methods #{:get}}]]))

backend/src/app/rpc/helpers.clj

@@ -11,7 +11,7 @@
    [app.common.data.macros :as dm]
    [app.http :as-alias http]
    [app.rpc :as-alias rpc]
-   [yetti.response :as-alias yres]))
+   [yetti.response :as yres]))
 
 ;; A utilty wrapper object for wrap service responses that does not
 ;; implements the IObj interface that make possible attach metadata to
@@ -78,3 +78,21 @@
                (let [exp (if (integer? max-age) max-age (inst-ms max-age))
                      val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
                  (update response ::yres/headers assoc "cache-control" val)))))
+
+(defn stream
+  "A convenience allias for yetti.response/stream-body"
+  [f]
+  (yres/stream-body f))
+
+(defn get-request
+  "Get http request from RPC params"
+  [params]
+  (assert (contains? params ::rpc/request-at) "rpc params required")
+  (-> (meta params)
+      (get ::http/request)))
+
+(defn get-auth-data
+  "Get http auth-data from RPC params"
+  [params]
+  (-> (get-request params)
+      (get ::http/auth-data)))

backend/src/app/rpc/management/subscription.clj

@@ -0,0 +1,183 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.management.subscription
+  (:require
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.schema.generators :as sg]
+   [app.common.time :as ct]
+   [app.db :as db]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.profile :as profile]
+   [app.rpc.doc :as doc]
+   [app.util.services :as sv]))
+
+;; ---- RPC METHOD: AUTHENTICATE
+
+(def ^:private
+  schema:authenticate-params
+  [:map {:title "authenticate-params"}])
+
+(def ^:private
+  schema:authenticate-result
+  [:map {:title "authenticate-result"}
+   [:profile-id ::sm/uuid]])
+
+(sv/defmethod ::auth
+  {::doc/added "2.12"
+   ::sm/params schema:authenticate-params
+   ::sm/result schema:authenticate-result}
+  [_ {:keys [::rpc/profile-id]}]
+  {:profile-id profile-id})
+
+;; ---- RPC METHOD: GET-CUSTOMER
+
+;; FIXME: move to app.common.time
+(def ^:private schema:timestamp
+  (sm/type-schema
+   {:type ::timestamp
+    :pred ct/inst?
+    :type-properties
+    {:title "inst"
+     :description "The same as :app.common.time/inst but encodes to epoch"
+     :error/message "should be an instant"
+     :gen/gen (->> (sg/small-int)
+                   (sg/fmap (fn [v] (ct/inst v))))
+     :decode/string #(some-> % ct/inst)
+     :encode/string #(some-> % inst-ms)
+     :decode/json #(some-> % ct/inst)
+     :encode/json #(some-> % inst-ms)}}))
+
+(def ^:private schema:subscription
+  [:map {:title "Subscription"}
+   [:id ::sm/text]
+   [:customer-id ::sm/text]
+   [:type [:enum
+           "unlimited"
+           "professional"
+           "enterprise"]]
+   [:status [:enum
+             "active"
+             "canceled"
+             "incomplete"
+             "incomplete_expired"
+             "past_due"
+             "paused"
+             "trialing"
+             "unpaid"]]
+
+   [:billing-period [:enum
+                     "month"
+                     "day"
+                     "week"
+                     "year"]]
+   [:quantity :int]
+   [:description [:maybe ::sm/text]]
+   [:created-at schema:timestamp]
+   [:start-date [:maybe schema:timestamp]]
+   [:ended-at [:maybe schema:timestamp]]
+   [:trial-end [:maybe schema:timestamp]]
+   [:trial-start [:maybe schema:timestamp]]
+   [:cancel-at [:maybe schema:timestamp]]
+   [:canceled-at [:maybe schema:timestamp]]
+   [:current-period-end [:maybe schema:timestamp]]
+   [:current-period-start [:maybe schema:timestamp]]
+   [:cancel-at-period-end :boolean]
+
+   [:cancellation-details
+    [:map {:title "CancellationDetails"}
+     [:comment [:maybe ::sm/text]]
+     [:reason [:maybe ::sm/text]]
+     [:feedback [:maybe
+                 [:enum
+                  "customer_service"
+                  "low_quality"
+                  "missing_feature"
+                  "other"
+                  "switched_service"
+                  "too_complex"
+                  "too_expensive"
+                  "unused"]]]]]])
+
+(def ^:private sql:get-customer-slots
+  "WITH teams AS (
+    SELECT tpr.team_id AS id,
+           tpr.profile_id AS profile_id
+      FROM team_profile_rel AS tpr
+     WHERE tpr.is_owner IS true
+       AND tpr.profile_id = ?
+  ), teams_with_slots AS (
+    SELECT tpr.team_id AS id,
+           count(*) AS total
+      FROM team_profile_rel AS tpr
+     WHERE tpr.team_id IN (SELECT id FROM teams)
+       AND tpr.can_edit IS true
+     GROUP BY 1
+     ORDER BY 2
+  )
+  SELECT max(total) AS total FROM teams_with_slots;")
+
+(defn- get-customer-slots
+  [cfg profile-id]
+  (let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
+    (:total result)))
+
+(def ^:private schema:get-customer-params
+  [:map])
+
+(def ^:private schema:get-customer-result
+  [:map
+   [:id ::sm/uuid]
+   [:name :string]
+   [:num-editors ::sm/int]
+   [:subscription {:optional true} schema:subscription]])
+
+(sv/defmethod ::get-customer
+  {::doc/added "2.12"
+   ::sm/params schema:get-customer-params
+   ::sm/result schema:get-customer-result}
+  [cfg {:keys [::rpc/profile-id]}]
+  (let [profile (profile/get-profile cfg profile-id)]
+    {:id (get profile :id)
+     :name (get profile :fullname)
+     :email (get profile :email)
+     :num-editors (get-customer-slots cfg profile-id)
+     :subscription (-> profile :props :subscription)}))
+
+
+;; ---- RPC METHOD: GET-CUSTOMER
+
+(def ^:private schema:update-customer-params
+  [:map
+   [:subscription [:maybe schema:subscription]]])
+
+(def ^:private schema:update-customer-result
+  [:map])
+
+(sv/defmethod ::update-customer
+  {::doc/added "2.12"
+   ::sm/params schema:update-customer-params
+   ::sm/result schema:update-customer-result}
+  [cfg {:keys [::rpc/profile-id subscription]}]
+  (let [{:keys [props] :as profile}
+        (profile/get-profile cfg profile-id ::db/for-update true)
+
+        props
+        (assoc props :subscription subscription)]
+
+    (l/dbg :hint "update customer"
+           :profile-id (str profile-id)
+           :subscription-type (get subscription :type)
+           :subscription-status (get subscription :status)
+           :subscription-quantity (get subscription :quantity))
+
+    (db/update! cfg :profile
+                {:props (db/tjson props)}
+                {:id profile-id}
+                {::db/return-keys false})
+
+    nil))

backend/src/app/rpc/quotes.clj

@@ -102,8 +102,7 @@
                     ::wrk/label "quotes-notification"
                     ::wrk/params {:to (vec admins)
                                   :subject subject
-                                  :body [{:type "text/plain"
-                                          :content content}]}}))))
+                                  :body content}}))))
 
 (defn- generic-check!
   [{:keys [::db/conn ::incr ::quote-sql ::count-sql ::default ::target] :or {incr 1} :as params}]

backend/src/app/setup/clock.clj

@@ -14,7 +14,9 @@
    [integrant.core :as ig])
   (:import
    java.time.Clock
-   java.time.Duration))
+   java.time.Duration
+   java.time.Instant
+   java.time.ZoneId))
 
 (defonce current
   (atom {:clock (Clock/systemDefaultZone)
@@ -36,6 +38,12 @@
   [_ _]
   (remove-watch current ::common))
 
+(defn fixed
+  "Get fixed clock, mainly used in tests"
+  [instant]
+  (Clock/fixed ^Instant (ct/inst instant)
+               ^ZoneId (ZoneId/of "Z")))
+
 (defn set-offset!
   [duration]
   (swap! current assoc :offset (some-> duration ct/duration)))

backend/src/app/srepl/cli.clj

@@ -61,8 +61,8 @@
                              :is-active is-active
                              :password password
                              :props {}}]
-               (->> (cmd.auth/create-profile! conn params)
-                    (cmd.auth/create-profile-rels! conn)))))))
+               (->> (cmd.auth/create-profile system params)
+                    (cmd.auth/create-profile-rels conn)))))))
 
 (defmethod exec-command "update-profile"
   [{:keys [fullname email password is-active]}]

backend/src/app/srepl/main.clj

@@ -25,6 +25,7 @@
    [app.db.sql :as-alias sql]
    [app.features.fdata :as fdata]
    [app.features.file-snapshots :as fsnap]
+   [app.http.session :as session]
    [app.loggers.audit :as audit]
    [app.main :as main]
    [app.msgbus :as mbus]
@@ -567,48 +568,12 @@
                                           :id file-id})))
     :deleted))
 
-(defn- restore-file*
-  [{:keys [::db/conn]} file-id]
-  (db/update! conn :file
-              {:deleted-at nil
-               :has-media-trimmed false}
-              {:id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-media-object
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-change
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-data
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  ;; Mark thumbnails to be deleted
-  (db/update! conn :file-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  (db/update! conn :file-tagged-object-thumbnail
-              {:deleted-at nil}
-              {:file-id file-id}
-              {::db/return-keys false})
-
-  :restored)
-
 (defn restore-file!
   "Mark a file and all related objects as not deleted"
   [file-id]
   (let [file-id (h/parse-uuid file-id)]
     (db/tx-run! main/system
-                (fn [system]
+                (fn [{:keys [::db/conn] :as system}]
                   (when-let [file (db/get* system :file
                                            {:id file-id}
                                            {::db/remove-deleted false
@@ -622,7 +587,9 @@
                                                      :cause "explicit call to restore-file!"}
                                     ::audit/tracked-at (ct/now)})
 
-                    (restore-file* system file-id))))))
+
+                    (#'files/restore-file conn file-id))
+                  :restored))))
 
 (defn delete-project!
   "Mark a project for deletion"
@@ -655,7 +622,7 @@
   (doseq [{:keys [id]} (db/query conn :file
                                  {:project-id project-id}
                                  {::sql/columns [:id]})]
-    (restore-file* cfg id))
+    (#'files/restore-file conn id))
 
   :restored)
 
@@ -877,10 +844,33 @@
                                                      :deleted-at deleted-at
                                                      :id id})))))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SSO
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn add-sso-config
+  [& {:keys [base-uri client-id client-secret domain]}]
+
+  (assert (and (string? base-uri) (str/starts-with? base-uri "http")) "expected a valid base-uri")
+  (assert (string? client-id) "expected a valid client-id")
+  (assert (string? client-secret) "expected a valid client-secret")
+  (assert (string? domain) "expected a valid domain")
+  (db/insert! main/system :sso-provider
+              {:id (uuid/next)
+               :type "oidc"
+               :client-id client-id
+               :client-secret client-secret
+               :domain domain
+               :base-uri base-uri}))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; MISC
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn decode-session-token
+  [token]
+  (session/decode-token main/system token))
+
 (defn instrument-var
   [var]
   (alter-var-root var (fn [f]

backend/src/app/tasks/file_gc.clj

@@ -218,6 +218,9 @@
     (when (or (nil? revn) (= revn (:revn file)))
       file)))
 
+;; FIXME: we should skip files that does not match the revn on the
+;; props and add proper schema for this task props
+
 (defn- process-file!
   [cfg {:keys [file-id] :as props}]
   (if-let [file (get-file cfg props)]

backend/src/app/tasks/file_gc_scheduler.clj

@@ -8,6 +8,7 @@
   "A maintenance task that is responsible of properly scheduling the
   file-gc task for all files that matches the eligibility threshold."
   (:require
+   [app.common.logging :as l]
    [app.common.time :as ct]
    [app.config :as cf]
    [app.db :as db]
@@ -21,25 +22,24 @@
           f.modified_at
      FROM file AS f
     WHERE f.has_media_trimmed IS false
-      AND f.modified_at < now() - ?::interval
+      AND f.modified_at < ?
       AND f.deleted_at IS NULL
     ORDER BY f.modified_at DESC
       FOR UPDATE OF f
      SKIP LOCKED")
 
-(defn- get-candidates
-  [{:keys [::db/conn ::min-age] :as cfg}]
-  (let [min-age (db/interval min-age)]
-    (db/plan conn [sql:get-candidates min-age] {:fetch-size 10})))
-
 (defn- schedule!
-  [cfg]
+  [{:keys [::db/conn] :as cfg} threshold]
   (let [total (reduce (fn [total {:keys [id modified-at revn]}]
-                        (let [params {:file-id id :modified-at modified-at :revn revn}]
+                        (let [params {:file-id id :revn revn}]
+                          (l/trc :hint "schedule"
+                                 :file-id (str id)
+                                 :revn revn
+                                 :modified-at (ct/format-inst modified-at))
                           (wrk/submit! (assoc cfg ::wrk/params params))
                           (inc total)))
                       0
-                      (get-candidates cfg))]
+                      (db/plan conn [sql:get-candidates threshold] {:fetch-size 10}))]
     {:processed total}))
 
 (defmethod ig/assert-key ::handler
@@ -53,12 +53,12 @@
 (defmethod ig/init-key ::handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (let [min-age (ct/duration (or (:min-age props) (::min-age cfg)))]
+    (let [threshold (-> (ct/duration (or (:min-age props) (::min-age cfg)))
+                        (ct/in-past))]
       (-> cfg
           (assoc ::db/rollback (:rollback? props))
-          (assoc ::min-age min-age)
           (assoc ::wrk/task :file-gc)
           (assoc ::wrk/priority 10)
           (assoc ::wrk/mark-retries 0)
-          (assoc ::wrk/delay 1000)
-          (db/tx-run! schedule!)))))
+          (assoc ::wrk/delay 10000)
+          (db/tx-run! schedule! threshold)))))

backend/src/app/util/template.clj

@@ -9,7 +9,7 @@
    [app.common.exceptions :as ex]
    [selmer.parser :as sp]))
 
-;; (sp/cache-off!)
+(sp/cache-off!)
 
 (defn render
   [path context]

backend/src/app/worker.clj

@@ -77,8 +77,8 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def ^:private sql:insert-new-task
-  "insert into task (id, name, props, queue, label, priority, max_retries, scheduled_at)
-   values (?, ?, ?, ?, ?, ?, ?, now() + ?)
+  "insert into task (id, name, props, queue, label, priority, max_retries, created_at, modified_at, scheduled_at)
+   values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    returning id")
 
 (def ^:private
@@ -88,7 +88,7 @@
       AND queue=?
       AND label=?
       AND status = 'new'
-      AND scheduled_at > now()")
+      AND scheduled_at > ?")
 
 (def ^:private schema:options
   [:map {:title "submit-options"}
@@ -111,17 +111,19 @@
 
   (check-options! options)
 
-  (let [duration  (ct/duration delay)
-        interval  (db/interval duration)
-        props     (db/tjson params)
-        id        (uuid/next)
-        tenant    (cf/get :tenant)
-        task      (d/name task)
-        queue     (str/ffmt "%:%" tenant (d/name queue))
-        conn      (db/get-connectable options)
-        deleted   (when dedupe
-                    (-> (db/exec-one! conn [sql:remove-not-started-tasks task queue label])
-                        :next.jdbc/update-count))]
+  (let [delay        (ct/duration delay)
+        now          (ct/now)
+        scheduled-at (-> (ct/plus now delay)
+                         (ct/truncate :millisecond))
+        props        (db/tjson params)
+        id           (uuid/next)
+        tenant       (cf/get :tenant)
+        task         (d/name task)
+        queue        (str/ffmt "%:%" tenant (d/name queue))
+        conn         (db/get-connectable options)
+        deleted      (when dedupe
+                       (-> (db/exec-one! conn [sql:remove-not-started-tasks task queue label now])
+                           (db/get-update-count)))]
 
     (l/trc :hint "submit task"
            :name task
@@ -129,11 +131,13 @@
            :queue queue
            :label label
            :dedupe (boolean dedupe)
-           :delay (ct/format-duration duration)
+           :delay (ct/format-duration delay)
            :replace (or deleted 0))
 
     (db/exec-one! conn [sql:insert-new-task id task props queue
-                        label priority max-retries interval])
+                        label priority max-retries
+                        now now scheduled-at])
+
     id))
 
 (defn invoke!

backend/src/app/worker/runner.clj

@@ -13,6 +13,7 @@
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.transit :as t]
+   [app.config :as cf]
    [app.db :as db]
    [app.metrics :as mtx]
    [app.redis :as rds]
@@ -60,7 +61,8 @@
 
 (defn get-error-context
   [_ item]
-  {:params item})
+  (-> (cf/logging-context)
+      (assoc :params item)))
 
 (defn- get-task
   [{:keys [::db/pool]} task-id]
@@ -131,6 +133,11 @@
   [{:keys [::id ::timeout] :as cfg} task-id scheduled-at]
   (loop [task (get-task cfg task-id)]
     (cond
+      (nil? task)
+      (l/wrn :hint "no task found on the database"
+             :runner-id id
+             :task-id task-id)
+
       (ex/exception? task)
       (if (or (db/connection-error? task)
               (db/serialization-error? task))
@@ -151,12 +158,9 @@
             (inst-ms (:scheduled-at task)))
       (l/wrn :hint "skiping task, rescheduled"
              :task-id task-id
-             :runner-id id)
-
-      (nil? task)
-      (l/wrn :hint "no task found on the database"
              :runner-id id
-             :task-id task-id)
+             :scheduled-at (ct/format-inst (:scheduled-at task))
+             :expected-scheduled-at (ct/format-inst scheduled-at))
 
       :else
       (let [result (run-task cfg task)]
@@ -177,7 +181,8 @@
                           {:error explain
                            :status "retry"
                            :modified-at now
-                           :scheduled-at (ct/plus now delay)
+                           :scheduled-at (-> (ct/plus now delay)
+                                             (ct/truncate :millisecond))
                            :retry-num nretry}
                           {:id (:id task)})
               nil))
@@ -213,6 +218,7 @@
                          :payload payload)))
               (catch Throwable cause
                 (l/err :hint "unable to decode payload"
+                       ::l/context (cf/logging-context)
                        :payload payload
                        :length (alength ^String/1 payload)
                        :cause cause))))
@@ -224,11 +230,11 @@
                "failed"    (handle-task-failure result)
                "completed" (handle-task-completion result)
                (throw (IllegalArgumentException.
-                       (str "invalid status received: " status))))))
+                       (str "invalid status received: '" status "'"))))))
 
           (run-task-loop [[task-id scheduled-at]]
             (loop [result (run-task! cfg task-id scheduled-at)]
-              (when-let [cause (process-result result)]
+              (when-let [cause (some-> result process-result)]
                 (if (or (db/connection-error? cause)
                         (db/serialization-error? cause))
                   (do
@@ -236,9 +242,9 @@
                            :cause cause)
                     (px/sleep timeout)
                     (recur result))
-                  (do
-                    (l/err :hint "unhandled exception on processing task result"
-                           :cause cause))))))]
+                  (l/err :hint "unhandled exception on processing task result"
+                         ::l/context (cf/logging-context)
+                         :cause cause)))))]
 
     (try
       (let [key         (str/ffmt "penpot.worker.queue:%" queue)
@@ -254,11 +260,14 @@
         (if (rds/timeout-exception? cause)
           (do
             (l/err :hint "redis pop operation timeout, consider increasing redis timeout (will retry in some instants)"
+                   ::l/context (cf/logging-context)
                    :timeout timeout
                    :cause cause)
             (px/sleep timeout))
 
-          (l/err :hint "unhandled exception" :cause cause))))))
+          (l/err :hint "unhandled exception"
+                 ::l/context (cf/logging-context)
+                 :cause cause))))))
 
 (defn- start-thread!
   [{:keys [::id ::queue ::wrk/tenant] :as cfg}]
@@ -284,6 +293,7 @@
                  :queue queue))
         (catch Throwable cause
           (l/err :hint "unexpected exception"
+                 ::l/context (cf/logging-context)
                  :id id
                  :queue queue
                  :cause cause))

backend/test/backend_tests/email_sending_test.clj

@@ -22,4 +22,4 @@
     (t/is (contains? result :body))
     (t/is (contains? result :to))
     #_(t/is (contains? result :reply-to))
-    (t/is (vector? (:body result)))))
+    (t/is (map? (:body result)))))

backend/test/backend_tests/helpers.clj

@@ -104,13 +104,8 @@
                      (assoc-in [:app.rpc/methods :app.setup/templates] templates)
                      (dissoc :app.srepl/server
                              :app.http/server
-                             :app.http/router
-                             :app.auth.oidc.providers/google
-                             :app.auth.oidc.providers/gitlab
-                             :app.auth.oidc.providers/github
-                             :app.auth.oidc.providers/generic
+                             :app.http/route
                              :app.setup/templates
-                             :app.auth.oidc/routes
                              :app.http.oauth/handler
                              :app.notifications/handler
                              :app.loggers.mattermost/reporter
@@ -182,10 +177,10 @@
                         :is-demo false}
                        params)]
      (db/run! system
-              (fn [{:keys [::db/conn]}]
+              (fn [{:keys [::db/conn] :as cfg}]
                 (->> params
-                     (cmd.auth/create-profile! conn)
-                     (cmd.auth/create-profile-rels! conn)))))))
+                     (cmd.auth/create-profile cfg)
+                     (cmd.auth/create-profile-rels conn)))))))
 
 (defn create-project*
   ([i params] (create-project* *system* i params))
@@ -549,6 +544,44 @@
       (io/copy r sw)
       (.toString sw))))
 
+(defn parse-sse
+  [content]
+  (let [state
+        (reduce (fn [{:keys [events data event id] :as state} line]
+                  (cond
+                    ;; empty line → dispatch event if we have data
+                    (str/blank? line)
+                    (if (seq data)
+                      (-> state
+                          (update :events conj {:event (or event "message")
+                                                :data (-> (str/join "\n" data))})
+                          (assoc :data [] :event nil))
+                      state)
+
+                    ;; comment line (starts with :)
+                    (str/starts-with? line ":")
+                    state
+
+                    :else
+                    (let [[field raw-value] (str/split line #":" 2)
+                          value (some-> raw-value (str/replace #"^ " ""))]
+                      (case field
+                        "data"  (update state :data conj (or value ""))
+                        "event" (assoc state :event value)
+                        ;; ignore retry and unknown fields
+                        state))))
+                {:events [] :data [] :event nil}
+                (str/split content #"\r?\n"))
+
+        ;; handle unterminated last event (no trailing blank line)
+        state (if (seq (:data state))
+                (update state :events conj
+                        {:event (or (:event state) "message")
+                         :data (str/join "\n" (:data state))})
+                state)]
+
+    (:events state)))
+
 (defn consume-sse
   [callback]
   (let [{:keys [::yres/status ::yres/body ::yres/headers] :as response} (callback {})
@@ -558,12 +591,9 @@
     (try
       (px/exec! :virtual #(rcp/write-body-to-stream body nil output))
       (into []
-            (map (fn [event]
-                   (let [[item1 item2] (re-seq #"(.*): (.*)\n?" event)]
-
-                     [(keyword (nth item1 2))
-                      (tr/decode-str (nth item2 2))])))
-            (-> (slurp' input)
-                (str/split "\n\n")))
+            (map (fn [{:keys [event data]}]
+                   [(keyword event)
+                    (tr/decode-str data)]))
+            (parse-sse (slurp' input)))
       (finally
         (.close input)))))

backend/test/backend_tests/http_management_test.clj

@@ -22,17 +22,6 @@
 (t/use-fixtures :once th/state-init)
 (t/use-fixtures :each th/database-reset)
 
-
-(t/deftest authenticate-method
-  (let [profile  (th/create-profile* 1)
-        token    (#'sess/gen-token th/*system* {:profile-id (:id profile)})
-        request  {:params {:token token}}
-        response (#'mgmt/authenticate th/*system* request)]
-
-    (t/is (= 200 (::yres/status response)))
-    (t/is (= "authentication" (-> response ::yres/body :iss)))
-    (t/is (= (:id profile) (-> response ::yres/body :uid)))))
-
 (t/deftest get-customer-method
   (let [profile  (th/create-profile* 1)
         request  {:params {:id (:id profile)}}
@@ -89,7 +78,3 @@
 
       (let [subs' (-> response ::yres/body :subscription)]
         (t/is (= subs' subs))))))
-
-
-
-

backend/test/backend_tests/http_middleware_access_token_test.clj

@@ -1,57 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns backend-tests.http-middleware-access-token-test
-  (:require
-   [app.db :as db]
-   [app.http.access-token]
-   [app.main :as-alias main]
-   [app.rpc :as-alias rpc]
-   [app.rpc.commands.access-token]
-   [app.tokens :as tokens]
-   [backend-tests.helpers :as th]
-   [clojure.test :as t]
-   [mockery.core :refer [with-mocks]]))
-
-(t/use-fixtures :once th/state-init)
-(t/use-fixtures :each th/database-reset)
-
-(t/deftest soft-auth-middleware
-  (let [profile (th/create-profile* 1)
-        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
-
-        request (volatile! nil)
-        handler (#'app.http.access-token/wrap-soft-auth
-                 (fn [req] (vreset! request req))
-                 th/*system*)]
-
-    (with-mocks [m1 {:target 'app.http.access-token/get-token
-                     :return nil}]
-      (handler {})
-      (t/is (= {} @request)))
-
-    (with-mocks [m1 {:target 'app.http.access-token/get-token
-                     :return (:token token)}]
-      (handler {})
-
-      (let [token-id (get @request :app.http.access-token/id)]
-        (t/is (= token-id (:id token)))))))
-
-(t/deftest authz-middleware
-  (let [profile (th/create-profile* 1)
-        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
-        request (volatile! {})
-        handler (#'app.http.access-token/wrap-authz
-                 (fn [req] (vreset! request req))
-                 th/*system*)]
-
-    (handler nil)
-    (t/is (nil? @request))
-
-    (handler {:app.http.access-token/id (:id token)})
-    (t/is (= #{} (:app.http.access-token/perms @request)))
-    (t/is (= (:id profile) (:app.http.access-token/profile-id @request)))))
-

backend/test/backend_tests/http_middleware_test.clj

@@ -0,0 +1,135 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns backend-tests.http-middleware-test
+  (:require
+   [app.common.time :as ct]
+   [app.db :as db]
+   [app.http :as-alias http]
+   [app.http.access-token]
+   [app.http.middleware :as mw]
+   [app.http.session :as session]
+   [app.main :as-alias main]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.access-token]
+   [app.tokens :as tokens]
+   [backend-tests.helpers :as th]
+   [clojure.test :as t]
+   [mockery.core :refer [with-mocks]]
+   [yetti.request :as yreq]
+   [yetti.response :as yres]))
+
+(t/use-fixtures :once th/state-init)
+(t/use-fixtures :each th/database-reset)
+
+(defrecord DummyRequest [headers cookies]
+  yreq/IRequestCookies
+  (get-cookie [_ name]
+    {:value (get cookies name)})
+
+  yreq/IRequest
+  (get-header [_ name]
+    (get headers name)))
+
+(t/deftest auth-middleware-1
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {"authorization" "Token aaaa"} {}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :token token-type))
+      (t/is (= "aaaa" token))
+      (t/is (nil? claims)))))
+
+(t/deftest auth-middleware-2
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {"authorization" "Bearer aaaa"} {}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :bearer token-type))
+      (t/is (= "aaaa" token))
+      (t/is (nil? claims)))))
+
+(t/deftest auth-middleware-3
+  (let [request (volatile! nil)
+        handler (#'app.http.middleware/wrap-auth
+                 (fn [req] (vreset! request req))
+                 {})]
+
+    (handler (->DummyRequest {} {}))
+    (t/is (nil? (::http/auth-data @request)))
+
+    (handler (->DummyRequest {} {"auth-token" "foobar"}))
+
+    (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
+      (t/is (= :cookie token-type))
+      (t/is (= "foobar" token))
+      (t/is (nil? claims)))))
+
+(t/deftest shared-key-auth
+  (let [handler (#'app.http.middleware/wrap-shared-key-auth
+                 (fn [req] {::yres/status 200})
+                 "secret-key")]
+
+    (let [response (handler (->DummyRequest {} {}))]
+      (t/is (= 403 (::yres/status response))))
+
+    (let [response (handler (->DummyRequest {"x-shared-key" "secret-key2"} {}))]
+      (t/is (= 403 (::yres/status response))))
+
+    (let [response (handler (->DummyRequest {"x-shared-key" "secret-key"} {}))]
+      (t/is (= 200 (::yres/status response))))))
+
+(t/deftest access-token-authz
+  (let [profile (th/create-profile* 1)
+        token   (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
+        handler (#'app.http.access-token/wrap-authz identity th/*system*)]
+
+    (let [response (handler nil)]
+      (t/is (nil? response)))
+
+    (let [response (handler {::http/auth-data {:type :token :token "foobar" :claims {:tid (:id token)}}})]
+      (t/is (= #{} (:app.http.access-token/perms response)))
+      (t/is (= (:id profile) (:app.http.access-token/profile-id response))))))
+
+(t/deftest session-authz
+  (let [cfg      th/*system*
+        manager  (session/inmemory-manager)
+        profile  (th/create-profile* 1)
+        handler  (-> (fn [req] req)
+                     (#'session/wrap-authz  {::session/manager manager})
+                     (#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
+                                      :cookie (partial session/decode-token cfg)}))
+
+        session  (->> (session/create-session manager {:profile-id (:id profile)
+                                                       :user-agent "user agent"})
+                      (#'session/assign-token cfg))
+
+        response (handler (->DummyRequest {} {"auth-token" (:token session)}))
+
+        {:keys [token claims] token-type :type}
+        (get response ::http/auth-data)]
+
+    (t/is (= :cookie token-type))
+    (t/is (= (:token session) token))
+    (t/is (= "authentication" (:iss claims)))
+    (t/is (= "penpot" (:aud claims)))
+    (t/is (= (:id session) (:sid claims)))
+    (t/is (= (:id profile) (:uid claims)))))

backend/test/backend_tests/rpc_doc_test.clj

@@ -23,7 +23,7 @@
   (smt/check!
    (smt/for [context (->> sg/int
                           (sg/fmap (fn [_]
-                                     (rpc.doc/prepare-openapi-context (::rpc/methods th/*system*)))))]
+                                     (#'rpc.doc/openapi-context (::rpc/methods th/*system*)))))]
      (try
        (json/encode context)
        true

backend/test/backend_tests/rpc_file_test.clj

@@ -19,6 +19,7 @@
    [app.http :as http]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.files :as files]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -142,126 +143,112 @@
           (t/is (= 0 (count result))))))))
 
 (t/deftest file-gc-with-fragments
-  (letfn [(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-            (let [params {::th/type :update-file
-                          ::rpc/profile-id profile-id
-                          :id file-id
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :features cfeat/supported-features
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
-              (:result out)))]
+  (let [profile (th/create-profile* 1)
+        file    (th/create-file* 1 {:profile-id (:id profile)
+                                    :project-id (:default-project-id profile)
+                                    :is-shared false})
 
-    (let [profile (th/create-profile* 1)
-          file    (th/create-file* 1 {:profile-id (:id profile)
-                                      :project-id (:default-project-id profile)
-                                      :is-shared false})
+        page-id  (uuid/random)
+        shape-id (uuid/random)]
 
-          page-id  (uuid/random)
-          shape-id (uuid/random)]
+    ;; Preventive file-gc
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file) :revn (:revn file)})))
 
-      ;; Preventive file-gc
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file) :revn (:revn file)})))
+    ;; Check the number of fragments before adding the page
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments before adding the page
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 2 (count rows))))
+    ;; Add page
+    (update-file!
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-page
+       :name "test"
+       :id page-id}])
 
-      ;; Add page
-      (update-file!
-       :file-id (:id file)
-       :profile-id (:id profile)
-       :revn 0
-       :vern 0
-       :changes
-       [{:type :add-page
-         :name "test"
-         :id page-id}])
+    ;; Check the number of fragments before adding the page
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 3 (count rows))))
 
-      ;; Check the number of fragments before adding the page
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 3 (count rows))))
+    ;; The file-gc should mark for remove unused fragments
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      ;; The file-gc should mark for remove unused fragments
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+    ;; Check the number of fragments
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 5 (count rows)))
+      (t/is (= 3 (count (filterv :deleted-at rows)))))
 
-      ;; Check the number of fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 5 (count rows)))
-        (t/is (= 3 (count (filterv :deleted-at rows)))))
+    ;; The objects-gc should remove unused fragments
+    (let [res (th/run-task! :objects-gc {})]
+      (t/is (= 3 (:processed res))))
 
-      ;; The objects-gc should remove unused fragments
-      (let [res (th/run-task! :objects-gc {})]
-        (t/is (= 3 (:processed res))))
+    ;; Check the number of fragments
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 2 (count rows))))
+    ;; Add shape to page that should add a new fragment
+    (update-file!
+     :file-id (:id file)
+     :profile-id (:id profile)
+     :revn 0
+     :vern 0
+     :changes
+     [{:type :add-obj
+       :page-id page-id
+       :id shape-id
+       :parent-id uuid/zero
+       :frame-id uuid/zero
+       :components-v2 true
+       :obj (cts/setup-shape
+             {:id shape-id
+              :name "image"
+              :frame-id uuid/zero
+              :parent-id uuid/zero
+              :type :rect})}])
 
-      ;; Add shape to page that should add a new fragment
-      (update-file!
-       :file-id (:id file)
-       :profile-id (:id profile)
-       :revn 0
-       :vern 0
-       :changes
-       [{:type :add-obj
-         :page-id page-id
-         :id shape-id
-         :parent-id uuid/zero
-         :frame-id uuid/zero
-         :components-v2 true
-         :obj (cts/setup-shape
-               {:id shape-id
-                :name "image"
-                :frame-id uuid/zero
-                :parent-id uuid/zero
-                :type :rect})}])
+    ;; Check the number of fragments
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 3 (count rows))))
 
-      ;; Check the number of fragments
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 3 (count rows))))
+    ;; The file-gc should mark for remove unused fragments
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      ;; The file-gc should mark for remove unused fragments
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+    ;; The objects-gc should remove unused fragments
+    (let [res (th/run-task! :objects-gc {})]
+      (t/is (= 3 (:processed res))))
 
-      ;; The objects-gc should remove unused fragments
-      (let [res (th/run-task! :objects-gc {})]
-        (t/is (= 3 (:processed res))))
+    ;; Check the number of fragments;
+    (let [rows (th/db-query :file-data {:file-id (:id file)
+                                        :type "fragment"
+                                        :deleted-at nil})]
+      (t/is (= 2 (count rows))))
 
-      ;; Check the number of fragments;
-      (let [rows (th/db-query :file-data {:file-id (:id file)
-                                          :type "fragment"
-                                          :deleted-at nil})]
-        (t/is (= 2 (count rows))))
+    ;; Lets proceed to delete all changes
+    (th/db-delete! :file-change {:file-id (:id file)})
+    (th/db-delete! :file-data {:file-id (:id file) :type "snapshot"})
 
-      ;; Lets proceed to delete all changes
-      (th/db-delete! :file-change {:file-id (:id file)})
-      (th/db-delete! :file-data {:file-id (:id file) :type "snapshot"})
+    (th/db-update! :file
+                   {:has-media-trimmed false}
+                   {:id (:id file)})
 
-      (th/db-update! :file
-                     {:has-media-trimmed false}
-                     {:id (:id file)})
+    ;; The file-gc should remove fragments related to changes
+    ;; snapshots previously deleted.
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      ;; The file-gc should remove fragments related to changes
-      ;; snapshots previously deleted.
-      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
+    ;; Check the number of fragments;
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      ;; (pp/pprint rows)
+      (t/is (= 4 (count rows)))
+      (t/is (= 2 (count (remove :deleted-at rows)))))
 
-      ;; Check the number of fragments;
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        ;; (pp/pprint rows)
-        (t/is (= 4 (count rows)))
-        (t/is (= 2 (count (remove :deleted-at rows)))))
+    (let [res (th/run-task! :objects-gc {})]
+      (t/is (= 2 (:processed res))))
 
-      (let [res (th/run-task! :objects-gc {})]
-        (t/is (= 2 (:processed res))))
-
-      (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
-        (t/is (= 2 (count rows)))))))
+    (let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
+      (t/is (= 2 (count rows))))))
 
 (t/deftest file-gc-with-thumbnails
   (letfn [(add-file-media-object [& {:keys [profile-id file-id]}]
@@ -279,20 +266,6 @@
 
               ;; (th/print-result! out)
               (t/is (nil? (:error out)))
-              (:result out)))
-
-          (update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
-            (let [params {::th/type :update-file
-                          ::rpc/profile-id profile-id
-                          :id file-id
-                          :session-id (uuid/random)
-                          :revn revn
-                          :vern 0
-                          :features cfeat/supported-features
-                          :changes changes}
-                  out    (th/command! params)]
-              ;; (th/print-result! out)
-              (t/is (nil? (:error out)))
               (:result out)))]
 
     (let [storage (:app.storage/storage th/*system*)
@@ -1893,3 +1866,125 @@
 
       (t/is (= (:id file-2) (:file-id (get rows 0))))
       (t/is (nil? (:deleted-at (get rows 0)))))))
+
+(t/deftest deleted-files-permanently-delete
+  (let [prof    (th/create-profile* 1 {:is-active true})
+        team-id (:default-team-id prof)
+        proj-id (:default-project-id prof)
+        file-id (uuid/next)
+        now     (ct/inst "2025-10-31T00:00:00Z")]
+
+    (binding [ct/*clock* (clock/fixed now)]
+      (let [data {::th/type :create-file
+                  ::rpc/profile-id (:id prof)
+                  :project-id proj-id
+                  :id file-id
+                  :name "foobar"
+                  :is-shared false
+                  :components-v2 true}
+            out (th/command! data)]
+
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+
+        (let [result (:result out)]
+          (t/is (= (:name data) (:name result)))
+          (t/is (= proj-id (:project-id result)))))
+
+      (let [data {::th/type :delete-file
+                  :id file-id
+                  ::rpc/profile-id (:id prof)}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (t/is (nil? (:result out))))
+
+      ;; get deleted files
+      (let [data {::th/type :get-team-deleted-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [[row1 :as result] (:result out)]
+          (t/is (= 1 (count result)))
+          (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
+
+      (let [data {::th/type :permanently-delete-team-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id
+                  :ids #{file-id}}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [result (:result out)]
+          (t/is (= (:ids data) result)))
+
+        (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
+          (t/is (= (:deleted-at row) now)))))))
+
+(t/deftest deleted-files-restore
+  (let [prof    (th/create-profile* 1 {:is-active true})
+        team-id (:default-team-id prof)
+        proj-id (:default-project-id prof)
+        file-id (uuid/next)
+        now     (ct/inst "2025-10-31T00:00:00Z")]
+
+    (binding [ct/*clock* (clock/fixed now)]
+      (let [data {::th/type :create-file
+                  ::rpc/profile-id (:id prof)
+                  :project-id proj-id
+                  :id file-id
+                  :name "foobar"
+                  :is-shared false
+                  :components-v2 true}
+            out (th/command! data)]
+
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+
+        (let [result (:result out)]
+          (t/is (= (:name data) (:name result)))
+          (t/is (= proj-id (:project-id result)))))
+
+      (let [data {::th/type :delete-file
+                  :id file-id
+                  ::rpc/profile-id (:id prof)}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (t/is (nil? (:result out))))
+
+      ;; get deleted files
+      (let [data {::th/type :get-team-deleted-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [[row1 :as result] (:result out)]
+          (t/is (= 1 (count result)))
+          (t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
+          (t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
+          (t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
+
+      (let [data {::th/type :restore-deleted-team-files
+                  ::rpc/profile-id (:id prof)
+                  :team-id team-id
+                  :ids #{file-id}}
+            out (th/command! data)]
+        ;; (th/print-result! out)
+        (t/is (nil? (:error out)))
+        (let [result (:result out)]
+          (t/is (fn? result))
+
+          (let [events (th/consume-sse result)]
+            ;; (pp/pprint events)
+            (t/is (= 2 (count events)))
+            (t/is (= :end (first (last events))))
+            (t/is (= (:ids data) (last (last events)))))))
+
+      (let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
+        (t/is (nil? (:deleted-at row)))))))

backend/test/backend_tests/rpc_project_test.clj

@@ -104,7 +104,8 @@
       ;; (th/print-result! out)
       (t/is (nil? (:error out)))
       (let [result (:result out)]
-        (t/is (= 1 (count result)))))))
+        (t/is (= 1 (count (remove :deleted-at result))))
+        (t/is (= 2 (count result)))))))
 
 (t/deftest permissions-checks-create-project
   (let [profile1 (th/create-profile* 1)
@@ -207,7 +208,8 @@
       ;; (th/print-result! out)
       (t/is (nil? (:error out)))
       (let [result (:result out)]
-        (t/is (= 1 (count result)))))
+        (t/is (= 2 (count result)))
+        (t/is (= 1 (count (remove :deleted-at result))))))
 
     ;; run permanent deletion (should be noop)
     (let [result (th/run-task! :objects-gc {})]

common/src/app/common/data.cljc

@@ -1024,6 +1024,35 @@
       :clj
       (sort comp-fn items))))
 
+(defn reorder
+  "Reorder a vector by moving one of their items from some position to some space between positions.
+   It clamps the position numbers to a valid range."
+  [v from-pos to-space-between-pos]
+  (let [max-space-pos  (count v)
+        max-prop-pos   (dec max-space-pos)
+
+        from-pos             (max 0 (min max-prop-pos from-pos))
+        to-space-between-pos (max 0 (min max-space-pos to-space-between-pos))]
+
+    (if (= from-pos to-space-between-pos)
+      v
+      (let [elem         (nth v from-pos)
+            without-elem (-> []
+                             (into (subvec v 0 from-pos))
+                             (into (subvec v (inc from-pos))))
+            insert-pos   (if (< from-pos to-space-between-pos)
+                           (dec to-space-between-pos)
+                           to-space-between-pos)]
+        (-> []
+            (into (subvec without-elem 0 insert-pos))
+            (into [elem])
+            (into (subvec without-elem insert-pos)))))))
+
+(defn invert-map
+  "Returns a map with keys and values swapped.
+   If the input map has duplicate values, later entries overwrite earlier ones."
+  [m]
+  (into {} (map (fn [[k v]] [v k]) m)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; String Functions

common/src/app/common/files/builder.cljc

@@ -485,6 +485,13 @@
         (commit-change change1)
         (commit-change change2))))
 
+(defn add-tokens-lib
+  [state tokens-lib]
+  (-> state
+      (commit-change
+       {:type :set-tokens-lib
+        :tokens-lib tokens-lib})))
+
 (defn delete-shape
   [file id]
   (commit-change

common/src/app/common/files/changes.cljc

@@ -371,7 +371,7 @@
    [:set-tokens-lib
     [:map {:title "SetTokensLib"}
      [:type [:= :set-tokens-lib]]
-     [:tokens-lib ::sm/any]]]  ;; TODO: we should define a plain object schema for tokens-lib
+     [:tokens-lib ctob/schema:tokens-lib]]]
 
    [:set-token
     [:map {:title "SetTokenChange"}
@@ -517,8 +517,7 @@
    (when verify?
      (check-changes items))
 
-   (binding [*touched-changes* (volatile! #{})
-             cts/*wasm-sync* true]
+   (binding [*touched-changes* (volatile! #{})]
      (let [result (reduce #(or (process-change %1 %2) %1) data items)
            result (reduce process-touched-change result @*touched-changes*)]
        ;; Validate result shapes (only on the backend)

common/src/app/common/files/changes_builder.cljc

@@ -638,6 +638,7 @@
                                   (reduce add-undo-change-shape $ ids)))
          (apply-changes-local)))))
 
+;; FIXME: PERFORMANCE
 (defn resize-parents
   [changes ids]
   (assert-page-id! changes)

common/src/app/common/files/helpers.cljc

@@ -72,9 +72,11 @@
        (= :bool (dm/get-prop shape :type))))
 
 (defn text-shape?
-  [shape]
-  (and (some? shape)
-       (= :text (dm/get-prop shape :type))))
+  ([shape]
+   (and (some? shape)
+        (= :text (dm/get-prop shape :type))))
+  ([objects id]
+   (text-shape? (get objects id))))
 
 (defn rect-shape?
   [shape]

common/src/app/common/files/migrations.cljc

@@ -1357,38 +1357,6 @@
         (update :pages-index d/update-vals update-container)
         (d/update-when :components d/update-vals update-container))))
 
-(defmethod migrate-data "0004-clean-shadow-color"
-  [data _]
-  (let [decode-color (sm/decoder types.color/schema:color sm/json-transformer)
-
-        clean-shadow-color
-        (fn [color]
-          (let [ref-id   (get color :id)
-                ref-file (get color :file-id)]
-            (-> (d/without-qualified color)
-                (select-keys [:opacity :color :gradient :image :ref-id :ref-file])
-                (cond-> ref-id
-                  (assoc :ref-id ref-id))
-                (cond-> ref-file
-                  (assoc :ref-file ref-file))
-                (decode-color))))
-
-        clean-shadow
-        (fn [shadow]
-          (update shadow :color clean-shadow-color))
-
-        update-object
-        (fn [object]
-          (d/update-when object :shadow #(mapv clean-shadow %)))
-
-        update-container
-        (fn [container]
-          (d/update-when container :objects d/update-vals update-object))]
-
-    (-> data
-        (update :pages-index d/update-vals update-container)
-        (d/update-when :components d/update-vals update-container))))
-
 (defmethod migrate-data "0005-deprecate-image-type"
   [data _]
   (letfn [(update-object [object]
@@ -1413,17 +1381,27 @@
 (defmethod migrate-data "0006-fix-old-texts-fills"
   [data _]
   (letfn [(fix-fills [node]
-            (let [fills (if (and (not (seq (:fills node)))
-                                 (or (some? (:fill-color node))
-                                     (some? (:fill-opacity node))
-                                     (some? (:fill-color-gradient node))))
-                          [(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient
-                                                              :fill-color-ref-id :fill-color-ref-file]))]
-                          (:fills node))]
-              (-> node
-                  (assoc :fills fills)
-                  (dissoc :fill-color :fill-opacity :fill-color-gradient
-                          :fill-color-ref-id :fill-color-ref-file))))
+            (let [;; In the old format refs were strings
+                  sanitize-uuid
+                  (fn [o]
+                    (if (uuid? o)
+                      o
+                      (uuid/parse* o)))
+
+                  fills
+                  (if (and (not (seq (:fills node)))
+                           (or (some? (:fill-color node))
+                               (some? (:fill-opacity node))
+                               (some? (:fill-color-gradient node))))
+                    [(-> (select-keys node types.fills/fill-attrs)
+                         (update :fill-color-ref-file sanitize-uuid)
+                         (update :fill-color-ref-id sanitize-uuid)
+                         (d/without-nils))]
+                    (:fills node))]
+
+              (reduce dissoc
+                      (assoc node :fills fills)
+                      types.fills/fill-attrs)))
 
           (update-object [object]
             (if (cfh/text-shape? object)
@@ -1630,6 +1608,164 @@
   ;; as value; this migration fixes it.
   (d/update-when data :components d/update-vals d/without-nils))
 
+(defmethod migrate-data "0015-fix-text-attrs-blank-strings"
+  [data _]
+  ;; After making text validation more restrictive (using ::sm/text
+  ;; instead of :string), we need to fix text attributes that contain
+  ;; empty or blank strings. These should be replaced with default
+  ;; values from default-text-attrs.
+  (letfn [(blank-or-empty? [v]
+            (or (nil? v)
+                (and (string? v)
+                     (or (str/empty? v)
+                         (str/blank? v)))))
+
+          (get-default-value [attr]
+            (let [defaults types.text/default-text-attrs]
+              (case attr
+                ;; direction in content maps to text-direction in defaults
+                :direction (:text-direction defaults)
+                ;; For other attrs, get directly from defaults
+                (get defaults attr))))
+
+          (fix-text-attrs [node]
+            ;; These are the attributes that were changed to ::sm/text in the schema
+            (let [text-attrs [:font-family :font-size :font-style :font-weight
+                              :direction :text-decoration :text-transform]]
+              (reduce
+               (fn [node attr]
+                 (if (and (contains? node attr)
+                          (blank-or-empty? (get node attr)))
+                   ;; Replace blank/empty value with default
+                   (if-let [default-val (get-default-value attr)]
+                     (assoc node attr default-val)
+                     ;; If no default, remove the attribute
+                     (dissoc node attr))
+                   node))
+               node
+               text-attrs)))
+
+          (fix-position-data [position-data]
+            (mapv fix-text-attrs position-data))
+
+          (fix-text-content [content]
+            (types.text/transform-nodes types.text/is-content-node? fix-text-attrs content))
+
+          (update-shape [object]
+            (if (cfh/text-shape? object)
+              (-> object
+                  (d/update-when :content fix-text-content)
+                  (d/update-when :position-data fix-position-data))
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-shape))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
+(defmethod migrate-data "0015-clean-shadow-color"
+  [data _]
+  (let [decode-shadow-color
+        (sm/decoder ctss/schema:color sm/json-transformer)
+
+        clean-shadow-color
+        (fn [color]
+          (let [ref-id   (get color :id)
+                ref-file (get color :file-id)]
+            (-> (d/without-qualified color)
+                (select-keys ctss/color-attrs)
+                (cond-> ref-id
+                  (assoc :ref-id ref-id))
+                (cond-> ref-file
+                  (assoc :ref-file ref-file))
+                (decode-shadow-color)
+                (d/without-nils))))
+
+        clean-shadow
+        (fn [shadow]
+          (update shadow :color clean-shadow-color))
+
+        clean-xform
+        (comp
+         (keep clean-shadow)
+         (filter ctss/valid-shadow?))
+
+        update-object
+        (fn [object]
+          (d/update-when object :shadow #(into [] clean-xform %)))
+
+        update-container
+        (fn [container]
+          (d/update-when container :objects d/update-vals update-object))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
+;; Copy fills from position-data to text nodes when all text nodes lack fills,
+;; all position-data have fills, and the counts match
+(defmethod migrate-data "0016-copy-fills-from-position-data-to-text-node"
+  [data _]
+  (letfn [(get-text-nodes [content]
+            ;; Get all leaf text nodes from the content tree
+            (when content
+              (->> (types.text/node-seq types.text/is-text-node? content)
+                   (seq))))
+
+          (update-content [content fills-map]
+            ;; Transform the content tree to update text nodes with their corresponding fills
+            ;; fills-map is a map from text node to its fills
+            (types.text/transform-nodes
+             types.text/is-text-node?
+             (fn [text-node]
+               (if-let [fills (get fills-map text-node)]
+                 (assoc text-node :fills fills)
+                 text-node))
+             content))
+
+          (update-object [object]
+            (if (cfh/text-shape? object)
+              (let [content       (:content object)
+                    position-data (:position-data object)
+                    text-nodes    (get-text-nodes content)]
+
+                ;; Check if conditions are met:
+                ;; 1. Has at least one text node
+                ;; 2. All text nodes have no fills or empty fills
+                ;; 3. Has at least one position-data entry
+                ;; 4. All position-data have fills
+                ;; 5. The number of text nodes matches the number of position-data
+                (if (and (seq text-nodes)
+                         (seq position-data)
+                         (= (count text-nodes) (count position-data))
+                         (every? (fn [text-node]
+                                   (let [fills (:fills text-node)]
+                                     (or (nil? fills) (empty? fills))))
+                                 text-nodes)
+                         (every? (fn [pd]
+                                   (let [fills (:fills pd)]
+                                     (and (some? fills) (seq fills))))
+                                 position-data))
+
+                  ;; Apply the migration: create a map from each text node to its corresponding fills
+                  (let [fills-map (zipmap text-nodes (map :fills position-data))]
+                    (update object :content #(update-content % fills-map)))
+
+                  ;; Don't modify if conditions aren't met
+                  object))
+
+              ;; Not a text shape, return as-is
+              object))
+
+          (update-container [container]
+            (d/update-when container :objects d/update-vals update-object))]
+
+    (-> data
+        (update :pages-index d/update-vals update-container)
+        (d/update-when :components d/update-vals update-container))))
+
 (def available-migrations
   (into (d/ordered-set)
         ["legacy-2"
@@ -1689,7 +1825,6 @@
          "0002-clean-shape-interactions"
          "0003-fix-root-shape"
          "0003-convert-path-content-v2"
-         "0004-clean-shadow-color"
          "0005-deprecate-image-type"
          "0006-fix-old-texts-fills"
          "0008-fix-library-colors-v4"
@@ -1701,4 +1836,7 @@
          "0013-fix-component-path"
          "0013-clear-invalid-strokes-and-fills"
          "0014-fix-tokens-lib-duplicate-ids"
-         "0014-clear-components-nil-objects"]))
+         "0014-clear-components-nil-objects"
+         "0015-fix-text-attrs-blank-strings"
+         "0015-clean-shadow-color"
+         "0016-copy-fills-from-position-data-to-text-node"]))

common/src/app/common/files/tokens.cljc

@@ -44,7 +44,7 @@
   (let [attr? (set attributes)]
     (->> (remove (fn [[k v]]
                    (and (attr? k)
-                        (= v (token-identifier token))))
+                        (= v (or (token-identifier token) token))))
                  applied-tokens)
          (into {}))))
 

common/src/app/common/files/validate.cljc

@@ -466,19 +466,20 @@
         children   (map #(ctst/get-shape page %) shapes)
         prop-names (cfv/extract-properties-names (first children) (:data file))]
     (doseq [child children]
-      (if (not (ctk/is-variant? child))
-        (report-error :not-a-variant
-                      (str/ffmt "Shape % should be a variant" (:id child))
-                      child file page)
-        (do
-          (when (not= (:variant-id child) shape-id)
-            (report-error :invalid-variant-id
-                          (str/ffmt "Variant % has invalid variant-id %" (:id child) (:variant-id child))
-                          child file page))
-          (when (not= prop-names (cfv/extract-properties-names child (:data file)))
-            (report-error :invalid-variant-properties
-                          (str/ffmt "Variant % has invalid properties %" (:id child) (vec prop-names))
-                          child file page)))))))
+      (when child
+        (if (not (ctk/is-variant? child))
+          (report-error :not-a-variant
+                        (str/ffmt "Shape % should be a variant" (:id child))
+                        child file page)
+          (do
+            (when (not= (:variant-id child) shape-id)
+              (report-error :invalid-variant-id
+                            (str/ffmt "Variant % has invalid variant-id %" (:id child) (:variant-id child))
+                            child file page))
+            (when (not= prop-names (cfv/extract-properties-names child (:data file)))
+              (report-error :invalid-variant-properties
+                            (str/ffmt "Variant % has invalid properties %" (:id child) (vec prop-names))
+                            child file page))))))))
 
 (defn- check-variant
   "Shape is a variant, so
@@ -627,7 +628,8 @@
         main-component (if (:deleted component)
                          (dm/get-in component [:objects (:main-instance-id component)])
                          (ctst/get-shape component-page (:main-instance-id component)))]
-    (when-not (ctk/is-variant? main-component)
+    (when (and main-component
+               (not (ctk/is-variant? main-component)))
       (report-error :not-a-variant
                     (str/ffmt "Shape % should be a variant" (:id main-component))
                     main-component file component-page))))

common/src/app/common/files/variant.cljc

@@ -10,16 +10,23 @@
    [app.common.types.components-list :as ctcl]
    [app.common.types.variant :as ctv]))
 
-
 (defn find-variant-components
   "Find a list of the components thet belongs to this variant-id"
-  [data objects variant-id]
-  ;; We can't simply filter components, because we need to maintain the order
-  (->> (dm/get-in objects [variant-id :shapes])
-       (map #(dm/get-in objects [% :component-id]))
-       (map #(ctcl/get-component data % true))
-       reverse))
-
+  ([data variant-id]
+   (let [page-id (->> data
+                      :components
+                      vals
+                      (filter #(= (:variant-id %) variant-id))
+                      first
+                      :main-instance-page)
+         objects (dm/get-in data [:pages-index page-id :objects])]
+     (find-variant-components data objects variant-id)))
+  ([data objects variant-id]
+   ;; We can't simply filter components, because we need to maintain the order
+   (->> (dm/get-in objects [variant-id :shapes])
+        (map #(dm/get-in objects [% :component-id]))
+        (map #(ctcl/get-component data % true))
+        reverse)))
 
 (defn extract-properties-names
   [shape data]
@@ -28,7 +35,6 @@
        :variant-properties
        (map :name)))
 
-
 (defn extract-properties-values
   "Get a map of properties associated to their possible values"
   [data objects variant-id]
@@ -50,7 +56,6 @@
                       (get :objects))]
       (dm/get-in objects [variant-id :shapes]))))
 
-
 (defn is-secondary-variant?
   [component data]
   (let [shapes  (get-variant-mains component data)]

common/src/app/common/flags.cljc

@@ -33,7 +33,9 @@
     :login-with-ldap
     ;; Uses any generic authentication provider that implements OIDC protocol as credentials.
     :login-with-oidc
-    ;; Allows registration with Open ID
+    ;; Enables custom SSO flow
+    :login-with-custom-sso
+    ;; Allows registration with OIDC (takes effect only when general `registration` is disabled)
     :oidc-registration
     ;; This logs to console the invitation tokens. It's useful in case the SMTP is not configured.
     :log-invitation-tokens})
@@ -123,6 +125,7 @@
     :token-color
     :token-typography-types
     :token-typography-composite
+    :token-shadow
     :transit-readable-response
     :user-feedback
     ;; TODO: remove this flag.

common/src/app/common/geom/shapes.cljc

@@ -162,6 +162,7 @@
 (dm/export gtr/inverse-transform-matrix)
 (dm/export gtr/transform-rect)
 (dm/export gtr/calculate-geometry)
+(dm/export gtr/calculate-selrect)
 (dm/export gtr/update-group-selrect)
 (dm/export gtr/update-mask-selrect)
 (dm/export gtr/apply-transform)

common/src/app/common/logic/libraries.cljc

@@ -1642,7 +1642,8 @@
           (pcb/apply-changes-local)))))
 
 (defn- generate-update-tokens
-  [changes container dest-shape origin-shape touched omit-touched?]
+  [changes container dest-shape origin-shape touched omit-touched? valid-attrs]
+  ;; valid-attrs is a set of attrs to consider on the update. If it is nil, it will consider all the attrs
   (let [attrs (->> (seq (keys ctk/sync-attrs))
                    ;; We don't update the flex-child attrs
                    (remove #(= :layout-grid-cells %)))
@@ -1650,8 +1651,8 @@
         applied-tokens (reduce (fn [applied-tokens attr]
                                  (let [attr-group (get ctk/sync-attrs attr)
                                        token-attrs (cto/shape-attr->token-attrs attr)]
-                                   (if (not (and (touched attr-group)
-                                                 omit-touched?))
+                                   (if  (and (or (not omit-touched?) (not (touched attr-group)))
+                                             (or (empty? valid-attrs) (contains? valid-attrs attr)))
                                      (into applied-tokens token-attrs)
                                      applied-tokens)))
                                #{}
@@ -1808,7 +1809,7 @@
             :always
             (check-detached-main dest-shape origin-shape)
             :always
-            (generate-update-tokens container dest-shape origin-shape touched omit-touched?))
+            (generate-update-tokens container dest-shape origin-shape touched omit-touched? nil))
 
           (let [attr-group        (get ctk/sync-attrs attr)
                 ;; position-data is a special case because can be affected by
@@ -1991,6 +1992,12 @@
                ;; If the values are already equal, don't copy them
                (= (get previous-shape attr) (get current-shape attr))
 
+               ;; If the value is the same as the origin, don't copy it
+               (= (get previous-shape attr) (get origin-ref-shape attr))
+
+               ;; If the attr is not touched, don't copy it
+               (not (touched attr-group))
+
                ;; If both variants (origin and destiny) don't have the same value
                ;; for that attribute, don't copy it.
                ;; Exceptions: :points :selrect and :content can be different
@@ -2006,10 +2013,7 @@
                 (not= (get origin-ref-shape attr) (get current-shape attr)))
 
                ;; The :content attr cant't be copied to elements of different type
-               (and (= attr :content) (not= (:type previous-shape) (:type current-shape)))
-
-               ;; If the attr is not touched, don't copy it
-               (not (touched attr-group)))
+               (and (= attr :content) (not= (:type previous-shape) (:type current-shape))))
 
               ;; On texts, both text (the actual letters)
               ;; and attrs (bold, font, etc) are in the same attr :content.
@@ -2082,12 +2086,14 @@
           (recur (next attrs)
                  roperations'
                  uoperations'))
-        (cond-> changes
-          (> (count roperations) 1)
-          (add-update-attr-changes current-shape container roperations uoperations)
 
-          :always
-          (generate-update-tokens container current-shape previous-shape touched false))))))
+        (let [updated-attrs (into #{} (comp (filter #(= :set (:type %)))
+                                            (map :attr))
+                                  roperations)]
+          (cond-> changes
+            (> (count roperations) 1)
+            (-> (add-update-attr-changes current-shape container roperations uoperations)
+                (generate-update-tokens container current-shape previous-shape touched false updated-attrs))))))))
 
 (defn- propagate-attrs
   "Helper that puts the origin attributes (attrs) into dest but only if
@@ -2798,7 +2804,7 @@
 (defn generate-duplicate-changes
   "Prepare objects to duplicate: generate new id, give them unique names,
   move to the desired position, and recalculate parents and frames as needed."
-  [changes all-objects page ids delta libraries library-data file-id & {:keys [variant-props]}]
+  [changes all-objects page ids delta libraries library-data file-id & {:keys [variant-props alt-duplication?]}]
   (let [shapes         (map (d/getf all-objects) ids)
         unames         (volatile! (cfh/get-used-names (:objects page)))
         update-unames! (fn [new-name] (vswap! unames conj new-name))
@@ -2808,10 +2814,22 @@
         ;; we calculate a new one because the components will have created new shapes.
         ids-map        (into {} (map #(vector % (uuid/next))) all-ids)
 
+
+        ;; If there is an alt-duplication of a variant, change its parent to root
+        ;; so the copy is made as a child of root
+        ;; This is because inside a variant-container can't be a copy
+        shapes  (map (fn [shape]
+                       (if (and alt-duplication? (ctk/is-variant? shape))
+                         (assoc shape :parent-id uuid/zero :frame-id nil)
+                         shape))
+                     shapes)
+
+
         changes (-> changes
                     (pcb/with-page page)
                     (pcb/with-objects all-objects)
                     (pcb/with-library-data library-data))
+
         changes
         (->> shapes
              (reduce #(generate-duplicate-shape-change %1

common/src/app/common/logic/tokens.cljc

@@ -9,7 +9,7 @@
    [app.common.files.changes-builder :as pcb]
    [app.common.types.tokens-lib :as ctob]))
 
-(defn generate-update-active-sets
+(defn- generate-update-active-sets
   "Copy the active sets from the currently active themes and move them
   to the hidden token theme and update the theme with
   `update-theme-fn`.
@@ -28,12 +28,45 @@
         (pcb/set-token-theme (ctob/get-id hidden-theme)
                              hidden-theme'))))
 
+(defn generate-set-enabled-token-set
+  "Enable or disable a token set at `set-name` in `tokens-lib` without modifying a user theme."
+  [changes tokens-lib set-name enabled?]
+  (if enabled?
+    (generate-update-active-sets changes tokens-lib #(ctob/enable-set % set-name))
+    (generate-update-active-sets changes tokens-lib #(ctob/disable-set % set-name))))
+
 (defn generate-toggle-token-set
-  "Toggle a token set at `set-name` in `tokens-lib` without modifying a
-  user theme."
+  "Toggle a token set at `set-name` in `tokens-lib` without modifying a user theme."
   [changes tokens-lib set-name]
   (generate-update-active-sets changes tokens-lib #(ctob/toggle-set % set-name)))
 
+(defn- generate-update-active-token-theme
+  "Change the active state of a theme in `tokens-lib`. If after the change there is
+   any active theme other than the hidden one, deactivate the hidden theme."
+  [changes tokens-lib update-fn]
+  (let [active-token-themes (some-> tokens-lib
+                                    (update-fn)
+                                    (ctob/get-active-theme-paths))
+        active-token-themes' (if (= active-token-themes #{ctob/hidden-theme-path})
+                               active-token-themes
+                               (disj active-token-themes ctob/hidden-theme-path))]
+    (pcb/set-active-token-themes changes active-token-themes')))
+
+(defn generate-set-active-token-theme
+  "Activate or deactivate a token theme in `tokens-lib`."
+  [changes tokens-lib id active?]
+  (if active?
+    (generate-update-active-token-theme changes tokens-lib
+                                        #(ctob/activate-theme % id))
+    (generate-update-active-token-theme changes tokens-lib
+                                        #(ctob/deactivate-theme % id))))
+
+(defn generate-toggle-token-theme
+  "Toggle the active state of a token theme in `tokens-lib`."
+  [changes tokens-lib id]
+  (generate-update-active-token-theme changes tokens-lib
+                                      #(ctob/toggle-theme-active % id)))
+
 (defn toggle-token-set-group
   "Toggle a token set group at `group-path` in `tokens-lib` for a `tokens-lib-theme`."
   [group-path tokens-lib tokens-lib-theme]

common/src/app/common/logic/variant_properties.cljc

@@ -28,11 +28,7 @@
               (pcb/update-component
                changes (:id component)
                (fn [component]
-                 (d/update-in-when component [:variant-properties pos]
-                                   (fn [property]
-                                     (-> property
-                                         (assoc :name new-name)
-                                         (with-meta nil)))))
+                 (d/update-in-when component [:variant-properties pos] #(assoc % :name new-name)))
                {:apply-changes-local-library? true}))
             changes
             related-components)))
@@ -42,18 +38,21 @@
   [changes variant-id pos]
   (let [data               (pcb/get-library-data changes)
         objects            (pcb/get-objects changes)
-        related-components (cfv/find-variant-components data objects variant-id)]
-    (reduce (fn [changes component]
-              (let [props   (:variant-properties component)
-                    props   (d/remove-at-index props pos)
-                    main-id (:main-instance-id component)
-                    name    (ctv/properties-to-name props)]
-                (-> changes
-                    (pcb/update-component (:id component) #(assoc % :variant-properties props)
-                                          {:apply-changes-local-library? true})
-                    (pcb/update-shapes [main-id] #(assoc % :variant-name name)))))
-            changes
-            related-components)))
+        related-components (cfv/find-variant-components data objects variant-id)
+        props              (-> related-components first :variant-properties)]
+    (if (and (seq props) (<= 0 pos) (< pos (count props)))
+      (reduce (fn [changes component]
+                (let [props   (:variant-properties component)
+                      props   (d/remove-at-index props pos)
+                      main-id (:main-instance-id component)
+                      name    (ctv/properties-to-name props)]
+                  (-> changes
+                      (pcb/update-component (:id component) #(assoc % :variant-properties props)
+                                            {:apply-changes-local-library? true})
+                      (pcb/update-shapes [main-id] #(assoc % :variant-name name)))))
+              changes
+              related-components)
+      changes)))
 
 
 (defn generate-update-property-value
@@ -88,7 +87,7 @@
         related-components (cfv/find-variant-components data objects variant-id)]
     (reduce (fn [changes component]
               (let [props   (:variant-properties component)
-                    props   (ctv/reorder-by-moving-to-position props from-pos to-space-between-pos)
+                    props   (d/reorder props from-pos to-space-between-pos)
                     main-id (:main-instance-id component)
                     name    (ctv/properties-to-name props)]
                 (-> changes

common/src/app/common/logic/variants.cljc

@@ -67,7 +67,6 @@
       [[] {}]
       shapes))))
 
-
 (defn- keep-swapped-item
   "As part of the keep-touched process on a switch, given a child on the original
    copy that was swapped (orig-swapped-child), and its related shape on the new copy
@@ -88,7 +87,6 @@
         current-parent  (get objects (:parent-id related-shape-in-new))
         pos             (d/index-of (:shapes current-parent) (:id related-shape-in-new))]
 
-
     (-> (pcb/concat-changes before-changes changes)
 
         ;; Move the previous shape to the new parent
@@ -122,6 +120,29 @@
                     (subvec (vec ancestors) 1 (dec num-ancestors)))]
     (some ctk/get-swap-slot ancestors)))
 
+(defn- find-shape-ref-child-of
+  "Get the shape referenced by the shape-ref of the near main of the shape,
+   recursively repeated until find a shape-ref with parent-id as ancestor.
+   It will return the shape or nil if it doesn't found any"
+  [container libraries shape parent-id]
+  (let [ref-shape             (ctf/find-ref-shape nil container libraries shape
+                                                  :with-context? true)
+
+        ref-shape-container   (when ref-shape (:container (meta ref-shape)))
+        ref-shape-parents-set (when ref-shape
+                                (->> (cfh/get-parents-with-self (:objects ref-shape-container) (:id ref-shape))
+                                     (into #{} d/xf:map-id)))]
+
+    (if (or (nil? ref-shape) (contains? ref-shape-parents-set parent-id))
+      ref-shape
+      (find-shape-ref-child-of ref-shape-container libraries ref-shape parent-id))))
+
+(defn- add-touched-from-ref-chain
+  "Adds to the :touched attr of a shape the content of
+   the :touched of all its chain of ref shapes"
+  [container libraries shape]
+  (let [new-touched (ctf/get-touched-from-ref-chain-until-target-ref container libraries shape nil)]
+    (assoc shape :touched new-touched)))
 
 (defn generate-keep-touched
   "This is used as part of the switch process, when you switch from
@@ -141,7 +162,10 @@
         ;; Ignore children of swapped items, because
         ;; they will be moved without change when
         ;; managing their swapped ancestor
-        orig-touched       (->> (filter (comp seq :touched) original-shapes)
+        orig-touched       (->> original-shapes
+                                ;; Add to each shape also the touched of its ref chain
+                                (map #(add-touched-from-ref-chain container libraries %))
+                                (filter (comp seq :touched))
                                 (remove
                                  #(child-of-swapped? %
                                                      page-objects
@@ -158,20 +182,19 @@
 
         ;; The original-shape is in a copy. For the relation rules, we need the referenced
         ;; shape on the main component
-        orig-ref-shape     (ctf/find-ref-shape nil container libraries original-shape {:with-context? true})
-        orig-ref-objects   (:objects (:container (meta orig-ref-shape)))
+        orig-base-ref-shape (ctf/find-remote-shape container libraries original-shape {:with-context? true})
+        orig-ref-objects    (:objects (:container (meta orig-base-ref-shape)))
 
         ;; Adds a :shape-path attribute to the children of the orig-ref-shape,
         ;; that contains the type of its ancestors and its name
         o-ref-shapes-wp    (add-unique-path
-                            (reverse (cfh/get-children-with-self orig-ref-objects (:id orig-ref-shape)))
+                            (reverse (cfh/get-children-with-self orig-ref-objects (:id orig-base-ref-shape)))
                             orig-ref-objects
-                            (:id orig-ref-shape))
+                            (:id orig-base-ref-shape))
 
         ;; Creates a map to quickly find a child of the orig-ref-shape by its shape-path
         o-ref-shapes-p-map  (into {} (map (juxt :id :shape-path)) o-ref-shapes-wp)
 
-
         ;; Process each touched children of the original-shape
         [changes parents-of-swapped]
         (reduce
@@ -182,8 +205,7 @@
                  ;; orig-child-touched is in a copy. Get the referenced shape on the main component
                  ;; If there is a swap slot, we will get the referenced shape in another way
                  orig-ref-shape (when-not swap-slot
-                                  ;; TODO Maybe just get it from o-ref-shapes-wp
-                                  (ctf/find-ref-shape nil container libraries orig-child-touched))
+                                  (find-shape-ref-child-of container libraries orig-child-touched (:id orig-base-ref-shape)))
 
                  orig-ref-id    (if swap-slot
                                   ;; If there is a swap slot, find the referenced shape id
@@ -193,9 +215,11 @@
 
                  ;; Get the shape path of the referenced main
                  shape-path     (get o-ref-shapes-p-map orig-ref-id)
+
                  ;; Get its related shape in the children of new-shape: the one that
                  ;; has the same shape-path
                  related-shape-in-new  (get new-shapes-map shape-path)
+
                  parents-of-swapped (if related-shape-in-new
                                       (conj parent-of-swapped (:parent-id related-shape-in-new))
                                       parent-of-swapped)

common/src/app/common/schema.cljc

@@ -36,7 +36,7 @@
 
 (defn type
   [s]
-  (m/-type s))
+  (m/type s default-options))
 
 (defn properties
   [s]
@@ -46,6 +46,10 @@
   [s]
   (m/type-properties s))
 
+(defn children
+  [s]
+  (m/children s default-options))
+
 (defn schema
   [s]
   (if (schema? s)
@@ -127,9 +131,19 @@
 
 (defn keys
   "Given a map schema, return all keys as set"
-  [schema]
-  (->> (entries schema)
-       (into #{} xf:map-key)))
+  [schema']
+  (let [schema' (m/schema schema' default-options)]
+    (case (m/type schema')
+      :map
+      (->> (entries schema')
+           (into #{} xf:map-key))
+
+      :merge
+      (->> (m/children schema')
+           (mapcat m/entries)
+           (into #{} xf:map-key))
+
+      (throw (ex-info "not supported schema type" {:type (m/type schema')})))))
 
 (defn update-properties
   [s f & args]

common/src/app/common/test_helpers/variants.cljc

@@ -14,13 +14,14 @@
 
 (defn add-variant
   [file variant-label component1-label root1-label component2-label root2-label
-   & {:keys []}]
+   & {:keys [variant1-params variant2-params]
+      :or   {variant1-params {} variant2-params {}}}]
   (let [file (ths/add-sample-shape file variant-label :type :frame :is-variant-container true)
         variant-id (thi/id variant-label)]
 
     (-> file
-        (ths/add-sample-shape root2-label :type :frame :parent-label variant-label :variant-id variant-id :variant-name "Value2")
-        (ths/add-sample-shape root1-label :type :frame :parent-label variant-label :variant-id variant-id :variant-name "Value1")
+        (ths/add-sample-shape root2-label (assoc variant2-params :type :frame :parent-label variant-label :variant-id variant-id :variant-name "Value2"))
+        (ths/add-sample-shape root1-label (assoc variant1-params :type :frame :parent-label variant-label :variant-id variant-id :variant-name "Value1"))
         (thc/make-component component1-label root1-label)
         (thc/update-component component1-label {:variant-id variant-id :variant-properties [{:name "Property 1" :value "Value1"}]})
         (thc/make-component component2-label root2-label)
@@ -42,7 +43,8 @@
 
 (defn add-variant-with-child
   [file variant-label component1-label root1-label component2-label root2-label child1-label child2-label
-   & {:keys [child1-params child2-params]}]
+   & {:keys [child1-params child2-params]
+      :or   {child1-params {} child2-params {}}}]
   (let [file (ths/add-sample-shape file variant-label :type :frame :is-variant-container true)
         variant-id (thi/id variant-label)]
     (-> file

common/src/app/common/types/component.cljc

@@ -286,7 +286,7 @@
           (fn [touched]
             (into #{} (remove #(str/starts-with? (name %) "swap-slot-") touched)))))
 
-(defn get-component-root
+(defn get-deleted-component-root
   [component]
   (if (some? (:main-instance-id component))
     (get-in component [:objects (:main-instance-id component)])

common/src/app/common/types/file.cljc

@@ -32,6 +32,7 @@
    [app.common.types.typographies-list :as ctyl]
    [app.common.types.typography :as cty]
    [app.common.uuid :as uuid]
+   [clojure.set :as set]
    [cuerdas.core :as str]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -276,7 +277,7 @@
     (-> file-data
         (get-component-page component)
         (ctn/get-shape (:main-instance-id component)))
-    (ctk/get-component-root component)))
+    (ctk/get-deleted-component-root component)))
 
 (defn get-component-shape
   "Retrieve one shape in the component by id. If with-context? is true, add the
@@ -355,7 +356,7 @@
 
 (defn find-remote-shape
   "Recursively go back by the :shape-ref of the shape until find the correct shape of the original component"
-  [container libraries shape]
+  [container libraries shape & {:keys [with-context?] :or {with-context? false}}]
   (let [top-instance        (ctn/get-component-shape (:objects container) shape)
         component-file      (get-in libraries [(:component-file top-instance) :data])
         component           (ctkl/get-component component-file (:component-id top-instance) true)
@@ -375,8 +376,12 @@
     (if (nil? remote-shape)
       nil
       (if (nil? (:shape-ref remote-shape))
-        remote-shape
-        (find-remote-shape component-container libraries remote-shape)))))
+        (cond-> remote-shape
+          (and remote-shape with-context?)
+          (with-meta {:file {:id (:id file-data)
+                             :data file-data}
+                      :container component-container}))
+        (find-remote-shape component-container libraries remote-shape :with-context? with-context?)))))
 
 (defn direct-copy?
   "Check if the shape is in a direct copy of the component (i.e. the shape-ref points to shapes inside
@@ -901,7 +906,7 @@
     (println))
 
   (when (seq (:objects component))
-    (let [root (ctk/get-component-root component)]
+    (let [root (ctk/get-deleted-component-root component)]
       (dump-shape (:id root)
                   1
                   (:objects component)
@@ -1115,3 +1120,29 @@
 (defn set-base-font-size
   [file-data base-font-size]
   (assoc-in file-data [:options :base-font-size] base-font-size))
+
+
+;; Ref Chains
+(defn get-ref-chain-until-target-ref
+  "Returns a vector with the shape ref chain until target-ref, including itself"
+  [container libraries shape target-ref]
+  (loop [chain [shape]
+         current shape]
+    (if (= current target-ref)
+      chain
+      (if-let [ref (find-ref-shape nil container libraries current :with-context? true)]
+        (recur (conj chain ref) ref)
+        chain))))
+
+(defn get-touched-from-ref-chain-until-target-ref
+  "Returns a set with the :touched of all the items on the shape
+   ref chain until target-ref, including itself"
+  [container libraries shape target-ref]
+  (let [chain (get-ref-chain-until-target-ref container libraries shape target-ref)
+        more-touched (->> chain
+                          (map :touched)
+                          (remove nil?)
+                          (apply set/union)
+                          (remove ctk/swap-slot?)
+                          set)]
+    (set/union (or (:touched shape) #{}) more-touched)))

common/src/app/common/types/fills/impl.cljc

@@ -301,11 +301,17 @@
 
      IHeapWritable
      (-get-byte-size [_]
-       (- (.-byteLength dbuffer) 4))
+       ;; Include the 4-byte header with the fill count
+       (+ 4 (* size FILL-U8-SIZE)))
 
      (-write-to [_ heap offset]
-       (let [buffer' (.-buffer ^js/DataView dbuffer)]
-         (.set heap (js/Uint32Array. buffer' 4) offset)))
+       (let [buffer' (.-buffer ^js/DataView dbuffer)
+            ;; Calculate byte size: 4 bytes header + (size * FILL-U8-SIZE)
+             byte-size (+ 4 (* size FILL-U8-SIZE))
+             ;; Create Uint32Array with exact size needed (convert bytes to u32 elements)
+             u32-array (js/Uint32Array. buffer' 0 (/ byte-size 4))]
+         ;; Copy from offset 0 to include the header with fill count
+         (.set heap u32-array offset)))
 
      IBinaryFills
      (-get-image-ids [_]

common/src/app/common/types/modifiers.cljc

@@ -732,89 +732,89 @@
   [shape scale-text-content value]
   (update shape :content scale-text-content value))
 
+(defn scale-text-content
+  [content value]
+  (->> content
+       (txt/transform-nodes txt/is-text-node? (partial transform-text-node value))
+       (txt/transform-nodes txt/is-paragraph-node? (partial transform-paragraph-node value))))
+
+(defn apply-scale-content
+  [shape value]
+  ;; Scale can only be positive
+  (let [value (mth/abs value)]
+    (cond-> shape
+      (cfh/text-shape? shape)
+      (update-text-content scale-text-content value)
+
+      :always
+      (gsc/update-corners-scale value)
+
+      (d/not-empty? (:strokes shape))
+      (gss/update-strokes-width value)
+
+      (d/not-empty? (:shadow shape))
+      (gse/update-shadows-scale value)
+
+      (some? (:blur shape))
+      (gse/update-blur-scale value)
+
+      (ctl/flex-layout? shape)
+      (ctl/update-flex-scale value)
+
+      (ctl/grid-layout? shape)
+      (ctl/update-grid-scale value)
+
+      :always
+      (ctl/update-flex-child value))))
+
+(defn remove-children-set
+  [shapes children-to-remove]
+  (let [remove? (set children-to-remove)]
+    (d/removev remove? shapes)))
+
+(defn apply-modifier
+  [shape operation]
+  (let [type (dm/get-prop operation :type)]
+    (case type
+      :rotation
+      (let [rotation (dm/get-prop operation :value)]
+        (update shape :rotation #(mod (+ (or % 0) rotation) 360)))
+
+      :add-children
+      (let [value (dm/get-prop operation :value)
+            index (dm/get-prop operation :index)
+
+            shape
+            (if (some? index)
+              (update shape :shapes
+                      (fn [shapes]
+                        (if (vector? shapes)
+                          (d/insert-at-index shapes index value)
+                          (d/concat-vec shapes value))))
+              (update shape :shapes d/concat-vec value))]
+
+        ;; Remove duplication
+        (update shape :shapes #(into [] (apply d/ordered-set %))))
+
+      :remove-children
+      (let [value (dm/get-prop operation :value)]
+        (update shape :shapes remove-children-set value))
+
+      :scale-content
+      (let [value (dm/get-prop operation :value)]
+        (apply-scale-content shape value))
+
+      :change-property
+      (let [property (dm/get-prop operation :property)
+            value (dm/get-prop operation :value)]
+        (assoc shape property value))
+
+      ;; :default => no change to shape
+      shape)))
+
 (defn apply-structure-modifiers
   "Apply structure changes to a shape"
   [shape modifiers]
-  (letfn [(scale-text-content
-            [content value]
-            (->> content
-                 (txt/transform-nodes txt/is-text-node? (partial transform-text-node value))
-                 (txt/transform-nodes txt/is-paragraph-node? (partial transform-paragraph-node value))))
-
-          (apply-scale-content
-            [shape value]
-            ;; Scale can only be positive
-            (let [value (mth/abs value)]
-              (cond-> shape
-                (cfh/text-shape? shape)
-                (update-text-content scale-text-content value)
-
-                :always
-                (gsc/update-corners-scale value)
-
-                (d/not-empty? (:strokes shape))
-                (gss/update-strokes-width value)
-
-                (d/not-empty? (:shadow shape))
-                (gse/update-shadows-scale value)
-
-                (some? (:blur shape))
-                (gse/update-blur-scale value)
-
-                (ctl/flex-layout? shape)
-                (ctl/update-flex-scale value)
-
-                (ctl/grid-layout? shape)
-                (ctl/update-grid-scale value)
-
-                :always
-                (ctl/update-flex-child value))))]
-
-    (let [remove-children
-          (fn [shapes children-to-remove]
-            (let [remove? (set children-to-remove)]
-              (d/removev remove? shapes)))
-
-          apply-modifier
-          (fn [shape operation]
-            (let [type (dm/get-prop operation :type)]
-              (case type
-                :rotation
-                (let [rotation (dm/get-prop operation :value)]
-                  (update shape :rotation #(mod (+ (or % 0) rotation) 360)))
-
-                :add-children
-                (let [value (dm/get-prop operation :value)
-                      index (dm/get-prop operation :index)
-
-                      shape
-                      (if (some? index)
-                        (update shape :shapes
-                                (fn [shapes]
-                                  (if (vector? shapes)
-                                    (d/insert-at-index shapes index value)
-                                    (d/concat-vec shapes value))))
-                        (update shape :shapes d/concat-vec value))]
-
-                  ;; Remove duplication
-                  (update shape :shapes #(into [] (apply d/ordered-set %))))
-
-                :remove-children
-                (let [value (dm/get-prop operation :value)]
-                  (update shape :shapes remove-children value))
-
-                :scale-content
-                (let [value (dm/get-prop operation :value)]
-                  (apply-scale-content shape value))
-
-                :change-property
-                (let [property (dm/get-prop operation :property)
-                      value (dm/get-prop operation :value)]
-                  (assoc shape property value))
-
-                ;; :default => no change to shape
-                shape)))]
-
-      (as-> shape $
-        (reduce apply-modifier $ (dm/get-prop modifiers :structure-parent))
-        (reduce apply-modifier $ (dm/get-prop modifiers :structure-child))))))
+  (as-> shape $
+    (reduce apply-modifier $ (dm/get-prop modifiers :structure-parent))
+    (reduce apply-modifier $ (dm/get-prop modifiers :structure-child))))

common/src/app/common/types/path/impl.cljc

@@ -498,10 +498,10 @@
     [:map
      [:x schema:safe-number]
      [:y schema:safe-number]
-     [:c1x schema:safe-number]
-     [:c1y schema:safe-number]
-     [:c2x schema:safe-number]
-     [:c2y schema:safe-number]]]])
+     [:c1x {:optional true} schema:safe-number]
+     [:c1y {:optional true} schema:safe-number]
+     [:c2x {:optional true} schema:safe-number]
+     [:c2y {:optional true} schema:safe-number]]]])
 
 (def ^:private schema:segment
   [:multi {:title "PathSegment"

common/src/app/common/types/shape.cljc

@@ -36,8 +36,7 @@
    [app.common.uuid :as uuid]
    [clojure.set :as set]))
 
-(defonce ^:dynamic *wasm-sync* false)
-
+(defonce ^:dynamic *shape-changes* nil)
 (defonce wasm-enabled? false)
 (defonce wasm-create-shape (constantly nil))
 

common/src/app/common/types/shape/shadow.cljc

@@ -11,6 +11,14 @@
 
 (def styles #{:drop-shadow :inner-shadow})
 
+(def schema:color
+  [:merge {:title "ShadowColor"}
+   ctc/schema:color-attrs
+   ctc/schema:plain-color])
+
+(def color-attrs
+  (sm/keys schema:color))
+
 (def schema:shadow
   [:map {:title "Shadow"}
    [:id [:maybe ::sm/uuid]]
@@ -20,7 +28,7 @@
    [:blur ::sm/safe-number]
    [:spread ::sm/safe-number]
    [:hidden :boolean]
-   [:color ctc/schema:color]])
+   [:color schema:color]])
 
 (def check-shadow
   (sm/check-fn schema:shadow))

common/src/app/common/types/shape/text.cljc

@@ -34,13 +34,13 @@
           [:fills {:optional true}
            [:maybe
             [:vector {:gen/max 2}  schema:fill]]]
-          [:font-family {:optional true} :string]
-          [:font-size {:optional true} :string]
-          [:font-style {:optional true} :string]
-          [:font-weight {:optional true} :string]
-          [:direction {:optional true} :string]
-          [:text-decoration {:optional true} :string]
-          [:text-transform {:optional true} :string]
+          [:font-family {:optional true} ::sm/text]
+          [:font-size {:optional true} ::sm/text]
+          [:font-style {:optional true} ::sm/text]
+          [:font-weight {:optional true} ::sm/text]
+          [:direction {:optional true} ::sm/text]
+          [:text-decoration {:optional true} ::sm/text]
+          [:text-transform {:optional true} ::sm/text]
           [:typography-ref-id {:optional true} [:maybe ::sm/uuid]]
           [:typography-ref-file {:optional true} [:maybe ::sm/uuid]]
           [:children
@@ -51,13 +51,13 @@
              [:fills {:optional true}
               [:maybe
                [:vector {:gen/max 2} schema:fill]]]
-             [:font-family {:optional true} :string]
-             [:font-size {:optional true} :string]
-             [:font-style {:optional true} :string]
-             [:font-weight {:optional true} :string]
-             [:direction {:optional true} :string]
-             [:text-decoration {:optional true} :string]
-             [:text-transform {:optional true} :string]
+             [:font-family {:optional true} ::sm/text]
+             [:font-size {:optional true} ::sm/text]
+             [:font-style {:optional true} ::sm/text]
+             [:font-weight {:optional true} ::sm/text]
+             [:direction {:optional true} ::sm/text]
+             [:text-decoration {:optional true} ::sm/text]
+             [:text-transform {:optional true} ::sm/text]
              [:typography-ref-id {:optional true} [:maybe ::sm/uuid]]
              [:typography-ref-file {:optional true} [:maybe ::sm/uuid]]]]]]]]]]]]])
 
@@ -72,11 +72,11 @@
     [:width ::sm/safe-number]
     [:height ::sm/safe-number]
     [:fills [:vector {:gen/max 2} schema:fill]]
-    [:font-family {:optional true} :string]
-    [:font-size {:optional true} :string]
-    [:font-style {:optional true} :string]
-    [:font-weight {:optional true} :string]
+    [:font-family {:optional true} ::sm/text]
+    [:font-size {:optional true} ::sm/text]
+    [:font-style {:optional true} ::sm/text]
+    [:font-weight {:optional true} ::sm/text]
     [:rtl {:optional true} :boolean]
     [:text {:optional true} :string]
-    [:text-decoration {:optional true} :string]
-    [:text-transform {:optional true} :string]]])
+    [:text-decoration {:optional true} ::sm/text]
+    [:text-transform {:optional true} ::sm/text]]])

common/src/app/common/types/text.cljc

@@ -249,12 +249,16 @@
 (defn equal-attrs?
   "Given a text structure, and a map of attrs, check that all the internal attrs in
    paragraphs and sentences have the same attrs"
-  [item attrs]
-  (let [item-attrs (dissoc item :text :type :key :children)]
-    (and
-     (or (empty? item-attrs)
-         (= attrs (dissoc item :text :type :key :children)))
-     (every? #(equal-attrs? % attrs) (:children item)))))
+  ([item attrs]
+   ;; Ignore the root attrs of the content. We only want to check paragraphs and sentences
+   (equal-attrs? item attrs true))
+  ([item attrs ignore?]
+   (let [item-attrs (dissoc item :text :type :key :children)]
+     (and
+      (or ignore?
+          (empty? item-attrs)
+          (= attrs (dissoc item :text :type :key :children)))
+      (every? #(equal-attrs? % attrs false) (:children item))))))
 
 (defn get-first-paragraph-text-attrs
   "Given a content text structure, extract it's first paragraph

common/src/app/common/types/token.cljc

@@ -54,6 +54,7 @@
 (def token-type->dtcg-token-type
   {:boolean         "boolean"
    :border-radius   "borderRadius"
+   :shadow          "shadow"
    :color           "color"
    :dimensions      "dimension"
    :font-family     "fontFamilies"
@@ -77,7 +78,8 @@
       ;; Allow these properties to be imported with singular key names for backwards compability
       (assoc "fontWeight" :font-weight
              "fontSize" :font-size
-             "fontFamily" :font-family)))
+             "fontFamily" :font-family
+             "boxShadow" :shadow)))
 
 (def composite-token-type->dtcg-token-type
   "Custom set of conversion keys for composite typography token with `:line-height` available.
@@ -115,6 +117,12 @@
 
 (def border-radius-keys (schema-keys schema:border-radius))
 
+(def ^:private schema:shadow
+  [:map {:title "ShadowTokenAttrs"}
+   [:shadow {:optional true} token-name-ref]])
+
+(def shadow-keys (schema-keys schema:shadow))
+
 (def ^:private schema:stroke-width
   [:map
    [:stroke-width {:optional true} token-name-ref]])
@@ -271,6 +279,7 @@
 
 (def all-keys (set/union color-keys
                          border-radius-keys
+                         shadow-keys
                          stroke-width-keys
                          sizing-keys
                          opacity-keys
@@ -289,6 +298,7 @@
   [:merge {:title "AppliedTokens"}
    schema:tokens
    schema:border-radius
+   schema:shadow
    schema:sizing
    schema:spacing
    schema:rotation
@@ -300,6 +310,10 @@
    schema:text-decoration
    schema:dimensions])
 
+(defn token-attr?
+  [attr]
+  (contains? all-keys attr))
+
 (defn shape-attr->token-attrs
   ([shape-attr] (shape-attr->token-attrs shape-attr nil))
   ([shape-attr changed-sub-attr]
@@ -334,6 +348,7 @@
      (font-weight-keys shape-attr)     #{shape-attr :typography}
 
      (border-radius-keys shape-attr) #{shape-attr}
+     (shadow-keys shape-attr) #{shape-attr}
      (sizing-keys shape-attr) #{shape-attr}
      (opacity-keys shape-attr) #{shape-attr}
      (spacing-keys shape-attr) #{shape-attr}
@@ -361,6 +376,7 @@
              rotation-keys
              sizing-keys
              opacity-keys
+             shadow-keys
              position-attributes))
 
 (def rect-attributes
@@ -391,15 +407,15 @@
     :text    text-attributes
     nil))
 
-(defn appliable-attrs
+(defn appliable-attrs-for-shape
   "Returns intersection of shape `attributes` for `shape-type`."
   [attributes shape-type is-layout]
   (set/intersection attributes (shape-type->attributes shape-type is-layout)))
 
-(defn any-appliable-attr?
+(defn any-appliable-attr-for-shape?
   "Checks if `token-type` supports given shape `attributes`."
   [attributes token-type is-layout]
-  (seq (appliable-attrs attributes token-type is-layout)))
+  (d/not-empty? (appliable-attrs-for-shape attributes token-type is-layout)))
 
 ;; Token attrs that are set inside content blocks of text shapes, instead
 ;; at the shape level.
@@ -444,6 +460,30 @@
                                      spacing-margin-keys)]
     (unapply-token-id shape layout-item-attrs)))
 
+(def tokens-by-input
+  "A map from input name to applicable token for that input."
+  {:width #{:sizing :dimensions}
+   :height #{:sizing :dimensions}
+   :max-width #{:sizing :dimensions}
+   :max-height #{:sizing :dimensions}
+   :x #{:spacing :dimensions}
+   :y #{:spacing :dimensions}
+   :rotation #{:number :rotation}
+   :border-radius #{:border-radius :dimensions}
+   :row-gap #{:spacing :dimensions}
+   :column-gap #{:spacing :dimensions}
+   :horizontal-padding #{:spacing :dimensions}
+   :vertical-padding #{:spacing :dimensions}
+   :sided-paddings #{:spacing :dimensions}
+   :horizontal-margin #{:spacing :dimensions}
+   :vertical-margin #{:spacing :dimensions}
+   :sided-margins #{:spacing :dimensions}
+   :line-height #{:line-height :number}
+   :font-size #{:font-size}
+   :letter-spacing #{:letter-spacing}
+   :fill #{:color}
+   :stroke-color #{:color}})
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; TYPOGRAPHY
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -514,26 +554,11 @@
   [token-value]
   (string? token-value))
 
-(def tokens-by-input
-  "A map from input name to applicable token for that input."
-  {:width #{:sizing :dimensions}
-   :height #{:sizing :dimensions}
-   :max-width #{:sizing :dimensions}
-   :max-height #{:sizing :dimensions}
-   :x #{:spacing :dimensions}
-   :y #{:spacing :dimensions}
-   :rotation #{:number :rotation}
-   :border-radius #{:border-radius :dimensions}
-   :row-gap #{:spacing :dimensions}
-   :column-gap #{:spacing :dimensions}
-   :horizontal-padding #{:spacing :dimensions}
-   :vertical-padding #{:spacing :dimensions}
-   :sided-paddings #{:spacing :dimensions}
-   :horizontal-margin #{:spacing :dimensions}
-   :vertical-margin #{:spacing :dimensions}
-   :sided-margins #{:spacing :dimensions}
-   :line-height #{:line-height :number}
-   :font-size #{:font-size}
-   :letter-spacing #{:letter-spacing}
-   :fill #{:color}
-   :stroke-color #{:color}})
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SHADOW
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn shadow-composite-token-reference?
+  "Predicate if a shadow composite token is a reference value - a string pointing to another reference token."
+  [token-value]
+  (string? token-value))

common/src/app/common/types/tokens_lib.cljc

@@ -7,10 +7,11 @@
 (ns app.common.types.tokens-lib
   (:require
    #?(:clj [app.common.fressian :as fres])
-   #?(:clj [clojure.data.json :as json])
+   #?(:clj [clojure.data.json :as c.json])
    [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.files.helpers :as cfh]
+   [app.common.json :as json]
    [app.common.path-names :as cpn]
    [app.common.schema :as sm]
    [app.common.schema.generators :as sg]
@@ -198,8 +199,8 @@
      :tokens tokens})
 
   #?@(:clj
-      [json/JSONWriter
-       (-write [this writter options] (json/-write (datafy this) writter options))])
+      [c.json/JSONWriter
+       (-write [this writter options] (c.json/-write (datafy this) writter options))])
 
   INamedItem
   (get-id [_]
@@ -758,7 +759,7 @@
   (theme-active? [_ id] "predicate if token theme is active")
   (activate-theme [_ id] "adds theme from the active-themes")
   (deactivate-theme [_ id] "removes theme from the active-themes")
-  (toggle-theme-active? [_ id] "toggles theme in the active-themes")
+  (toggle-theme-active [_ id] "toggles theme in the active-themes")
   (get-hidden-theme [_] "get the hidden temporary theme"))
 
 (def schema:token-themes
@@ -901,6 +902,7 @@
   (delete-token [_ set-id token-id] "delete a token from a set")
   (toggle-set-in-theme [_ theme-id set-name] "toggle a set used / not used in a theme")
   (get-active-themes-set-names [_] "set of set names that are active in the the active themes")
+  (token-set-active? [_ set-name] "if a set is active in any of the active themes")
   (sets-at-path-all-active? [_ group-path] "compute active state for child sets at `group-path`.
 Will return a value that matches this schema:
 `:none`    None of the nested sets are active
@@ -911,6 +913,7 @@ Will return a value that matches this schema:
   (get-tokens [_ set-id] "return a map of tokens in the set, indexed by token-name"))
 
 (declare parse-multi-set-dtcg-json)
+(declare read-multi-set-dtcg)
 (declare export-dtcg-json)
 
 (deftype TokensLib [sets themes active-themes]
@@ -922,23 +925,23 @@ Will return a value that matches this schema:
      :active-themes active-themes})
 
   #?@(:clj
-      [json/JSONWriter
-       (-write [this writter options] (json/-write (export-dtcg-json this) writter options))])
+      [c.json/JSONWriter
+       (-write [this writter options] (c.json/-write (export-dtcg-json this) writter options))])
 
   ITokenSets
-  ;  Naming conventions:
-  ;  (TODO: this will disappear after refactoring the internal structure of TokensLib).
-  ;    Set name:                the complete name as a string, without prefix \"some-group/some-subgroup/some-set\".
-  ;    Set final name or fname: the last part of the name \"some-set\".
-  ;    Set path:                the groups part of the name, as a vector [\"some-group\" \"some-subgroup\"].
-  ;    Set path str:            the set path as a string \"some-group/some-subgroup\".
-  ;    Set full path:           the path including the fname, as a vector [\"some-group\", \"some-subgroup\", \"some-set\"].
-  ;    Set full path str:       the set full path as a string \"some-group/some-subgroup/some-set\".
-
-  ;    Set prefix:                        the two-characters prefix added to a full path item \"G-\" / \"S-\".
-  ;    Prefixed set path or ppath:        a path wit added prefixes [\"G-some-group\", \"G-some-subgroup\"].
-  ;    Prefixed set full path or pfpath:  a full path wit prefixes [\"G-some-group\", \"G-some-subgroup\", \"S-some-set\"].
-  ;    Prefixed set final name or pfname: a final name with prefix \"S-some-set\".
+  ;;  Naming conventions:
+  ;;  (TODO: this will disappear after refactoring the internal structure of TokensLib).
+  ;;    Set name:                the complete name as a string, without prefix \"some-group/some-subgroup/some-set\".
+  ;;    Set final name or fname: the last part of the name \"some-set\".
+  ;;    Set path:                the groups part of the name, as a vector [\"some-group\" \"some-subgroup\"].
+  ;;    Set path str:            the set path as a string \"some-group/some-subgroup\".
+  ;;    Set full path:           the path including the fname, as a vector [\"some-group\", \"some-subgroup\", \"some-set\"].
+  ;;    Set full path str:       the set full path as a string \"some-group/some-subgroup/some-set\".
+   ;
+  ;;    Set prefix:                        the two-characters prefix added to a full path item \"G-\" / \"S-\".
+  ;;    Prefixed set path or ppath:        a path wit added prefixes [\"G-some-group\", \"G-some-subgroup\"].
+  ;;    Prefixed set full path or pfpath:  a full path wit prefixes [\"G-some-group\", \"G-some-subgroup\", \"S-some-set\"].
+  ;;    Prefixed set final name or pfname: a final name with prefix \"S-some-set\".
   (add-set [_ token-set]
     (assert (token-set? token-set) "expected valid token-set")
     (let [path (get-set-prefixed-path token-set)]
@@ -1206,7 +1209,7 @@ Will return a value that matches this schema:
     (when-let [theme (get-theme this id)]
       (contains? active-themes (get-theme-path theme))))
 
-  (toggle-theme-active? [this id]
+  (toggle-theme-active [this id]
     (if (theme-active? this id)
       (deactivate-theme this id)
       (activate-theme this id)))
@@ -1270,6 +1273,10 @@ Will return a value that matches this schema:
           (mapcat :sets)
           (get-active-themes this)))
 
+  (token-set-active? [this set-name]
+    (let [set-names (get-active-themes-set-names this)]
+      (contains? set-names set-name)))
+
   (sets-at-path-all-active? [this group-path]
     (let [active-set-names (get-active-themes-set-names this)
           prefixed-path-str (set-group-path->set-group-prefixed-path-str group-path)]
@@ -1404,7 +1411,11 @@ Will return a value that matches this schema:
     ;; function that is declared but not defined; so we need to pass
     ;; an anonymous function and delegate the resolution to runtime
     {:encode/json #(export-dtcg-json %)
-     :decode/json #(parse-multi-set-dtcg-json %)}}))
+     :decode/json #(read-multi-set-dtcg %)
+     ;; FIXME: add better, more reallistic generator
+     :gen/gen (->> (sg/small-int)
+                   (sg/fmap (fn [_]
+                              (make-tokens-lib))))}}))
 
 (defn duplicate-set
   "Make a new set with a unique name, copying data from the given set in the lib."
@@ -1448,18 +1459,23 @@ Will return a value that matches this schema:
      ["value" :map]
      ["type" :string]]]))
 
+(def ^:private schema:dtcg-node
+  [:schema {:registry
+            {::simple-value
+             [:or :string :int :double]
+             ::value
+             [:or
+              [:ref ::simple-value]
+              [:vector ::simple-value]
+              [:map-of :string [:or
+                                [:ref ::simple-value]
+                                [:vector ::simple-value]]]]}}
+   [:map
+    ["$type" :string]
+    ["$value" [:ref ::value]]]])
+
 (def ^:private dtcg-node?
-  (sm/validator
-   [:or
-    [:map
-     ["$value" :string]
-     ["$type" :string]]
-    [:map
-     ["$value" [:sequential [:map ["$type" :string]]]]
-     ["$type" :string]]
-    [:map
-     ["$value" :map]
-     ["$type" :string]]]))
+  (sm/validator schema:dtcg-node))
 
 (defn- get-json-format
   "Searches through decoded token file and returns:
@@ -1552,6 +1568,46 @@ Will return a value that matches this schema:
     ;; Reference value
     value))
 
+(defn- convert-dtcg-shadow-composite
+  "Convert shadow token value from DTCG format to internal format."
+  [value]
+  (let [process-shadow (fn [shadow]
+                         (if (map? shadow)
+                           (let [legacy-shadow-type (get "type" shadow)]
+                             (-> shadow
+                                 (set/rename-keys {"x" :offsetX
+                                                   "offsetX" :offsetX
+                                                   "y" :offsetY
+                                                   "offsetY" :offsetY
+                                                   "blur" :blur
+                                                   "spread" :spread
+                                                   "color" :color
+                                                   "inset" :inset})
+                                 (update :inset #(cond
+                                                   (boolean? %) %
+                                                   (= "true" %) true
+                                                   (= "false" %) false
+                                                   (= legacy-shadow-type "innerShadow") true
+                                                   :else false))
+                                 (select-keys [:offsetX :offsetY :blur :spread :color :inset])))
+                           shadow))]
+    (cond
+      ;; Reference value - keep as string
+      (string? value)
+      value
+
+      ;; Array of shadows - process each
+      (sequential? value)
+      (mapv process-shadow value)
+
+      ;; Single shadow object - wrap in vector
+      (map? value)
+      [(process-shadow value)]
+
+      ;; Fallback - keep as is
+      :else
+      value)))
+
 (defn- flatten-nested-tokens-json
   "Convert a tokens tree in the decoded json fragment into a flat map,
    being the keys the token paths after joining the keys with '.'."
@@ -1574,6 +1630,7 @@ Will return a value that matches this schema:
                                          (case token-type
                                            :font-family (convert-dtcg-font-family token-value)
                                            :typography (convert-dtcg-typography-composite token-value)
+                                           :shadow (convert-dtcg-shadow-composite token-value)
                                            token-value))
                                        :description (get v "$description")))
              ;; Discard unknown type tokens
@@ -1605,6 +1662,43 @@ Will return a value that matches this schema:
   (assert (= (get-json-format decoded-json-tokens) :json-format/legacy) "expected a legacy format for `decoded-json-tokens`")
   (parse-single-set-dtcg-json set-name (legacy-json->dtcg-json decoded-json-tokens)))
 
+(def ^:private schema:multi-set-dtcg
+  "Schema for penpot multi-set dtcg json decoded data/
+
+  Mainly used for validate the structure of the incoming data before
+  proceed to parse it to our internal data structures."
+  [:schema {:registry
+            {::node
+             [:or
+              [:map-of :string [:ref ::node]]
+              schema:dtcg-node]}}
+   [:map
+    ["$themes" {:optional true}
+     [:vector
+      [:map {:title "Theme"}
+       ["id" {:optional true} :string]
+       ["name" :string]
+       ["description" :string]
+       ["isSource" :boolean]
+       ["selectedTokenSets"
+        [:map-of :string [:enum "enabled" "disabled"]]]]]]
+    ["$metadata" {:optional true}
+     [:map {:title "Metadata"}
+      ["tokenSetOrder" {:optional true} [:vector :string]]
+      ["activeThemes" {:optional true} [:vector :string]]
+      ["activeSets" {:optional true} [:vector :string]]]]
+
+    [:malli.core/default
+     [:map-of :string [:ref ::node]]]]])
+
+(def ^:private check-multi-set-dtcg-data
+  (sm/check-fn schema:multi-set-dtcg))
+
+(def ^:private decode-multi-set-dtcg-data
+  (sm/decoder schema:multi-set-dtcg
+              sm/json-transformer))
+
+;; FIXME: remove `-json` suffix
 (defn parse-multi-set-dtcg-json
   "Parse a decoded json file with multi sets in DTCG format into a TokensLib."
   [decoded-json]
@@ -1644,10 +1738,10 @@ Will return a value that matches this schema:
                              (uuid/next))
                      :name (get theme "name")
                      :group (get theme "group")
-                     :is-source (get theme "is-source")
+                     :is-source (or (get theme "isSource")
+                                    ;; NOTE: backward compatibility
+                                    (get theme "is-source"))
                      :external-id (get theme "id")
-                     :modified-at (some-> (get theme "modified-at")
-                                          (ct/inst))
                      :sets (into #{}
                                  (comp (map key)
                                        xf-normalize-set-name
@@ -1695,6 +1789,23 @@ Will return a value that matches this schema:
 
     library))
 
+(defn read-multi-set-dtcg
+  "Read penpot multi-set dctg tokens. Accepts string or JSON decoded
+  data (without any case transformation). Used as schema decoder and
+  in the SDK."
+  [data]
+  (let [data (if (string? data)
+               (json/decode data :key-fn identity)
+               data)
+        data #?(:cljs (if (object? data)
+                        (json/->clj data :key-fn identity)
+                        data)
+                :clj data)
+
+        data (decode-multi-set-dtcg-data data)]
+    (-> (check-multi-set-dtcg-data data)
+        (parse-multi-set-dtcg-json))))
+
 (defn- parse-multi-set-legacy-json
   "Parse a decoded json file with multi sets in legacy format into a TokensLib."
   [decoded-json]
@@ -1707,6 +1818,7 @@ Will return a value that matches this schema:
     (parse-multi-set-dtcg-json (merge other-data
                                       dtcg-sets-data))))
 
+;; FIXME: remove `-json` suffix
 (defn parse-decoded-json
   "Guess the format and content type of the decoded json file and parse it into a TokensLib.
    The `file-name` is used to determine the set name when the json file contains a single set."
@@ -1739,11 +1851,32 @@ Will return a value that matches this schema:
      {} value)
     value))
 
+(defn- shadow-token->dtcg-token
+  "Convert shadow token value from internal format to DTCG format."
+  [value]
+  (if (sequential? value)
+    (mapv (fn [shadow]
+            (if (map? shadow)
+              (-> shadow
+                  (set/rename-keys {:offsetX "offsetX"
+                                    :offsetY "offsetY"
+                                    :blur "blur"
+                                    :spread "spread"
+                                    :color "color"
+                                    :inset "inset"})
+                  (select-keys ["offsetX" "offsetY" "blur" "spread" "color" "inset"]))
+              shadow))
+          value)
+    value))
+
 (defn- token->dtcg-token [token]
   (cond-> {"$value" (cond-> (:value token)
                       ;; Transform typography token values
                       (= :typography (:type token))
-                      typography-token->dtcg-token)
+                      typography-token->dtcg-token
+                      ;; Transform shadow token values
+                      (= :shadow (:type token))
+                      shadow-token->dtcg-token)
            "$type" (cto/token-type->dtcg-token-type (:type token))}
     (:description token) (assoc "$description" (:description token))))
 
@@ -1755,15 +1888,15 @@ Will return a value that matches this schema:
          (filter #(and (instance? TokenTheme %)
                        (not (hidden-theme? %))))
          (map (fn [token-theme]
-                (let [theme-map (->> token-theme
-                                     (into {})
-                                     walk/stringify-keys)]
-                  (-> theme-map
-                      (set/rename-keys  {"sets" "selectedTokenSets"
-                                         "external-id" "id"})
-                      (update "selectedTokenSets" (fn [sets]
-                                                    (->> (for [s sets] [s "enabled"])
-                                                         (into {})))))))))
+                ;; NOTE: this probaly can be implemented as type method
+                (d/without-nils
+                 {"id" (:external-id token-theme)
+                  "name" (:name token-theme)
+                  "group" (:group token-theme)
+                  "description" (:description token-theme)
+                  "isSource" (:is-source token-theme)
+                  "selectedTokenSets" (reduce #(assoc %1 %2 "enabled") {} (:sets token-theme))}))))
+
         themes
         (->> (get-theme-tree tokens-lib)
              (tree-seq d/ordered-map? vals)
@@ -1773,29 +1906,34 @@ Will return a value that matches this schema:
         active-themes
         (-> (get-active-theme-paths tokens-lib)
             (disj hidden-theme-path))]
-    {:themes themes
-     :active-themes active-themes}))
+    [themes active-themes]))
 
 (defn export-dtcg-multi-file
   "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi json files each encoded in DTCG format."
   [tokens-lib]
-  (let [{:keys [themes active-themes]} (dtcg-export-themes tokens-lib)
-        sets (->> (get-sets tokens-lib)
-                  (map (fn [token-set]
-                         (let [name   (get-name token-set)
-                               tokens (get-tokens- token-set)]
-                           [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)])))
-                  (into {}))]
+  (let [[themes active-themes]
+        (dtcg-export-themes tokens-lib)
+
+        sets
+        (->> (get-sets tokens-lib)
+             (map (fn [token-set]
+                    (let [name   (get-name token-set)
+                          tokens (get-tokens- token-set)]
+                      [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)])))
+             (into {}))]
+
     (-> sets
         (assoc "$themes.json" themes)
-        (assoc "$metadata.json" {"tokenSetOrder" (get-set-names tokens-lib)
-                                 "activeThemes" active-themes
-                                 "activeSets" (get-active-themes-set-names tokens-lib)}))))
+        (assoc "$metadata.json"
+               {"tokenSetOrder" (get-set-names tokens-lib)
+                "activeThemes" active-themes
+                "activeSets" (get-active-themes-set-names tokens-lib)}))))
 
 (defn export-dtcg-json
   "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi sets json string in DTCG format."
   [tokens-lib]
-  (let [{:keys [themes active-themes]} (dtcg-export-themes tokens-lib)
+  (let [[themes active-themes]
+        (dtcg-export-themes tokens-lib)
 
         name-set-tuples
         (->> (get-set-tree tokens-lib)
@@ -1991,13 +2129,19 @@ Will return a value that matches this schema:
 
 #?(:clj
    (defn- migrate-to-v1-4
-     "Migrate the TokensLib data structure internals to v1.2 version; it
+     "Migrate the TokensLib data structure internals to v1.4 version; it
   expects input from v1.3 version"
      [params]
      (let [migrate-set-node
            (fn recurse [node]
-             (if (token-set-legacy? node)
+             (cond
+               (token-set-legacy? node)
                (make-token-set node)
+
+               (token-set? node)
+               node
+
+               :else
                (d/update-vals node recurse)))]
 
        (update params :sets d/update-vals migrate-set-node))))

common/src/app/common/types/variant.cljc

@@ -311,26 +311,16 @@
   [variant]
   (cpn/merge-path-item (:name variant) (str/replace (:variant-name variant) #", " " / ")))
 
-(defn reorder-by-moving-to-position
-  "Reorder a vector by moving one of their items from some position to some space between positions.
-   It clamps the position numbers to a valid range."
-  [props from-pos to-space-between-pos]
-  (let [max-space-pos  (count props)
-        max-prop-pos   (dec max-space-pos)
-
-        from-pos             (max 0 (min max-prop-pos from-pos))
-        to-space-between-pos (max 0 (min max-space-pos to-space-between-pos))]
-
-    (if (= from-pos to-space-between-pos)
-      props
-      (let [elem         (nth props from-pos)
-            without-elem (-> []
-                             (into (subvec props 0 from-pos))
-                             (into (subvec props (inc from-pos))))
-            insert-pos   (if (< from-pos to-space-between-pos)
-                           (dec to-space-between-pos)
-                           to-space-between-pos)]
-        (-> []
-            (into (subvec without-elem 0 insert-pos))
-            (into [elem])
-            (into (subvec without-elem insert-pos)))))))
+(defn find-boolean-pair
+  "Given a vector, return the map from 'bool-values' that contains both as keys.
+   Returns nil if none match."
+  [v]
+  (let [bool-values [{"on" true   "off" false}
+                     {"yes" true  "no" false}
+                     {"true" true "false" false}]]
+    (when (= (count v) 2)
+      (some (fn [b]
+              (when (and (contains? b (first v))
+                         (contains? b (last v)))
+                b))
+            bool-values))))

common/src/app/common/uri.cljc

@@ -8,6 +8,7 @@
   (:refer-clojure :exclude [uri?])
   (:require
    [app.common.data.macros :as dm]
+   [cuerdas.core :as str]
    [lambdaisland.uri :as u]
    [lambdaisland.uri.normalize :as un])
   #?(:clj
@@ -58,6 +59,14 @@
                   (map (fn [[k v]] [(key-fn k) (value-fn v)]))))
         (u/map->query-string))))
 
+(defn ensure-path-slash
+  [u]
+  (update (uri u) :path
+          (fn [path]
+            (if (str/ends-with? path "/")
+              path
+              (str path "/")))))
+
 #?(:clj
    (defmethod print-method lambdaisland.uri.URI [^URI this ^java.io.Writer writer]
      (.write writer "#")

common/test/common_tests/data_test.cljc

@@ -102,3 +102,14 @@
   (t/is (= (d/insert-at-index [:a :b :c :d] 1 [:a])
            [:a :b :c :d])))
 
+(t/deftest reorder
+  (let [v ["a" "b" "c" "d"]]
+    (t/is (= (d/reorder v 0 2) ["b" "a" "c" "d"]))
+    (t/is (= (d/reorder v 0 3) ["b" "c" "a" "d"]))
+    (t/is (= (d/reorder v 0 4) ["b" "c" "d" "a"]))
+    (t/is (= (d/reorder v 3 0) ["d" "a" "b" "c"]))
+    (t/is (= (d/reorder v 3 2) ["a" "b" "d" "c"]))
+    (t/is (= (d/reorder v 0 5) ["b" "c" "d" "a"]))
+    (t/is (= (d/reorder v 3 -1) ["d" "a" "b" "c"]))
+    (t/is (= (d/reorder v 5 -1) ["d" "a" "b" "c"]))
+    (t/is (= (d/reorder v -1 5) ["b" "c" "d" "a"]))))

common/test/common_tests/logic/variants_switch_test.cljc

@@ -18,6 +18,29 @@
 
 (t/use-fixtures :each thi/test-fixture)
 
+
+(t/deftest test-basic-switch
+  (let [;; ==== Setup
+        file      (-> (thf/sample-file :file1)
+                      (thv/add-variant
+                       :v01 :c01 :m01 :c02 :m02
+                       {:variant1-params {:width 5}
+                        :variant2-params  {:width 15}})
+
+                      (thc/instantiate-component :c01
+                                                 :copy01))
+        copy01 (ths/get-shape file :copy01)
+
+        ;; ==== Action
+        file'     (tho/swap-component file copy01 :c02 {:new-shape-label :copy02 :keep-touched? true})
+
+        copy01'   (ths/get-shape file' :copy02)]
+    (thf/dump-file file :keys [:width])
+    ;; The copy had width 5 before the switch
+    (t/is (= (:width copy01) 5))
+    ;; The rect has width 15 after the switch
+    (t/is (= (:width copy01') 15))))
+
 (t/deftest test-simple-switch
   (let [;; ==== Setup
         file      (-> (thf/sample-file :file1)
@@ -46,6 +69,40 @@
     (t/is (= (:width rect02') 15))))
 
 
+(t/deftest test-basic-switch-override
+  (let [;; ==== Setup
+        file      (-> (thf/sample-file :file1)
+                      (thv/add-variant
+                       :v01 :c01 :m01 :c02 :m02
+                       {:variant1-params {:width 5}
+                        :variant2-params  {:width 5}})
+
+                      (thc/instantiate-component :c01
+                                                 :copy01))
+        copy01 (ths/get-shape file :copy01)
+
+        ;; Change width of copy
+        page   (thf/current-page file)
+        changes (cls/generate-update-shapes (pcb/empty-changes nil (:id page))
+                                            #{(:id copy01)}
+                                            (fn [shape]
+                                              (assoc shape :width 25))
+                                            (:objects page)
+                                            {})
+
+        file   (thf/apply-changes file changes)
+        copy01 (ths/get-shape file :copy01)
+
+        ;; ==== Action
+        file'     (tho/swap-component file copy01 :c02 {:new-shape-label :copy02 :keep-touched? true})
+
+        copy01'   (ths/get-shape file' :copy02)]
+    (thf/dump-file file :keys [:width])
+    ;; The copy had width 25 before the switch
+    (t/is (= (:width copy01) 25))
+    ;; The override is keept: The copy still has width 25 after the switch
+    (t/is (= (:width copy01') 25))))
+
 (t/deftest test-switch-with-override
   (let [;; ==== Setup
         file      (-> (thf/sample-file :file1)
@@ -125,12 +182,10 @@
     ;; The rect has width 15 after the switch
     (t/is (= (:width rect02') 15))))
 
-
 (def font-size-path-paragraph [:content :children 0 :children 0 :font-size])
 (def font-size-path-0 [:content :children 0 :children 0 :children 0 :font-size])
 (def font-size-path-1 [:content :children 0 :children 0 :children 1 :font-size])
 
-
 (def text-path-0 [:content :children 0 :children 0 :children 0 :text])
 (def text-path-1 [:content :children 0 :children 0 :children 1 :text])
 (def text-lines-path [:content :children 0 :children 0 :children])
@@ -188,6 +243,8 @@
 
 
         ;; The copy clean has no overrides
+
+
         copy-clean       (ths/get-shape file :copy-clean)
         copy-clean-t     (ths/get-shape file :copy-clean-t)
 
@@ -209,6 +266,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-clean :c02 {:new-shape-label :copy-clean-2 :keep-touched? true})
                   (tho/swap-component copy-font-size :c02 {:new-shape-label :copy-font-size-2 :keep-touched? true})
@@ -234,6 +293,8 @@
     ;; Before the switch:
     ;;   * font size 14
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-clean-t font-size-path-0) "14"))
     (t/is (= (get-in copy-clean-t text-path-0) "hello world"))
 
@@ -248,6 +309,8 @@
     ;; Before the switch:
     ;;   * font size 25
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-font-size-t font-size-path-0) "25"))
     (t/is (= (get-in copy-font-size-t text-path-0) "hello world"))
 
@@ -306,6 +369,8 @@
 
 
         ;; The copy clean has no overrides
+
+
         copy-clean       (ths/get-shape file :copy-clean)
         copy-clean-t     (ths/get-shape file :copy-clean-t)
 
@@ -327,6 +392,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-clean :c02 {:new-shape-label :copy-clean-2 :keep-touched? true})
                   (tho/swap-component copy-font-size :c02 {:new-shape-label :copy-font-size-2 :keep-touched? true})
@@ -352,6 +419,8 @@
     ;; Before the switch:
     ;;   * font size 14
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-clean-t font-size-path-0) "14"))
     (t/is (= (get-in copy-clean-t text-path-0) "hello world"))
 
@@ -366,6 +435,8 @@
     ;; Before the switch:
     ;;   * font size 25
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-font-size-t font-size-path-0) "25"))
     (t/is (= (get-in copy-font-size-t text-path-0) "hello world"))
 
@@ -401,7 +472,6 @@
     (t/is (= (get-in copy-both-t' font-size-path-0) "50"))
     (t/is (= (get-in copy-both-t' text-path-0) "text overriden"))))
 
-
 (t/deftest test-switch-with-different-text-text-override
   (let [;; ==== Setup
         file (-> (thf/sample-file :file1)
@@ -423,6 +493,8 @@
 
 
         ;; The copy clean has no overrides
+
+
         copy-clean       (ths/get-shape file :copy-clean)
         copy-clean-t     (ths/get-shape file :copy-clean-t)
 
@@ -444,6 +516,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-clean :c02 {:new-shape-label :copy-clean-2 :keep-touched? true})
                   (tho/swap-component copy-font-size :c02 {:new-shape-label :copy-font-size-2 :keep-touched? true})
@@ -469,6 +543,8 @@
     ;; Before the switch:
     ;;   * font size 14
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-clean-t font-size-path-0) "14"))
     (t/is (= (get-in copy-clean-t text-path-0) "hello world"))
 
@@ -483,6 +559,8 @@
     ;; Before the switch:
     ;;   * font size 25
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-font-size-t font-size-path-0) "25"))
     (t/is (= (get-in copy-font-size-t text-path-0) "hello world"))
 
@@ -518,7 +596,6 @@
     (t/is (= (get-in copy-both-t' font-size-path-0) "25"))
     (t/is (= (get-in copy-both-t' text-path-0) "bye"))))
 
-
 (t/deftest test-switch-with-different-text-and-prop-text-override
   (let [;; ==== Setup
         file (-> (thf/sample-file :file1)
@@ -542,6 +619,8 @@
 
 
         ;; The copy clean has no overrides
+
+
         copy-clean       (ths/get-shape file :copy-clean)
         copy-clean-t     (ths/get-shape file :copy-clean-t)
 
@@ -563,6 +642,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-clean :c02 {:new-shape-label :copy-clean-2 :keep-touched? true})
                   (tho/swap-component copy-font-size :c02 {:new-shape-label :copy-font-size-2 :keep-touched? true})
@@ -588,6 +669,8 @@
     ;; Before the switch:
     ;;   * font size 14
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-clean-t font-size-path-0) "14"))
     (t/is (= (get-in copy-clean-t text-path-0) "hello world"))
 
@@ -602,6 +685,8 @@
     ;; Before the switch:
     ;;   * font size 25
     ;;   * text "hello world"
+
+
     (t/is (= (get-in copy-font-size-t font-size-path-0) "25"))
     (t/is (= (get-in copy-font-size-t text-path-0) "hello world"))
 
@@ -637,7 +722,6 @@
     (t/is (= (get-in copy-both-t' font-size-path-0) "50"))
     (t/is (= (get-in copy-both-t' text-path-0) "bye"))))
 
-
 (t/deftest test-switch-with-identical-structure-text-override
   (let [;; ==== Setup
         file (-> (thf/sample-file :file1)
@@ -657,6 +741,8 @@
 
 
         ;; Duplicate a text line in copy-structure-clean
+
+
         file                   (change-structure file :copy-structure-clean-t)
         copy-structure-clean   (ths/get-shape file :copy-structure-clean)
         copy-structure-clean-t (ths/get-shape file :copy-structure-clean-t)
@@ -678,6 +764,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-structure-clean :c02 {:new-shape-label :copy-structure-clean-2 :keep-touched? true})
                   (tho/swap-component copy-structure-unif :c02 {:new-shape-label :copy-structure-unif-2 :keep-touched? true})
@@ -763,7 +851,6 @@
     (t/is (= (get-in copy-structure-mixed-t' font-size-path-1) "40"))
     (t/is (= (get-in copy-structure-mixed-t' text-path-1) "new line 2"))))
 
-
 (t/deftest test-switch-with-different-prop-structure-text-override
   (let [;; ==== Setup
         file (-> (thf/sample-file :file1)
@@ -784,6 +871,8 @@
 
 
         ;; Duplicate a text line in copy-structure-clean
+
+
         file                   (change-structure file :copy-structure-clean-t)
         copy-structure-clean   (ths/get-shape file :copy-structure-clean)
         copy-structure-clean-t (ths/get-shape file :copy-structure-clean-t)
@@ -805,6 +894,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-structure-clean :c02 {:new-shape-label :copy-structure-clean-2 :keep-touched? true})
                   (tho/swap-component copy-structure-unif :c02 {:new-shape-label :copy-structure-unif-2 :keep-touched? true})
@@ -906,6 +997,8 @@
 
 
         ;; Duplicate a text line in copy-structure-clean
+
+
         file                   (change-structure file :copy-structure-clean-t)
         copy-structure-clean   (ths/get-shape file :copy-structure-clean)
         copy-structure-clean-t (ths/get-shape file :copy-structure-clean-t)
@@ -927,6 +1020,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-structure-clean :c02 {:new-shape-label :copy-structure-clean-2 :keep-touched? true})
                   (tho/swap-component copy-structure-unif :c02 {:new-shape-label :copy-structure-unif-2 :keep-touched? true})
@@ -971,6 +1066,8 @@
     ;; Second line:
     ;;   * font size 25
     ;;   * text "new line 2"
+
+
     (t/is (= (get-in copy-structure-unif-t font-size-path-0) "25"))
     (t/is (= (get-in copy-structure-unif-t text-path-0) "new line 1"))
     (t/is (= (get-in copy-structure-unif-t font-size-path-1) "25"))
@@ -992,6 +1089,8 @@
     ;; Before the switch, second line:
     ;;   * font size 40
     ;;   * text "new line 2"
+
+
     (t/is (= (get-in copy-structure-mixed-t font-size-path-0) "35"))
     (t/is (= (get-in copy-structure-mixed-t text-path-0) "new line 1"))
     (t/is (= (get-in copy-structure-mixed-t font-size-path-1) "40"))
@@ -1025,6 +1124,8 @@
 
 
         ;; Duplicate a text line in copy-structure-clean
+
+
         file                   (change-structure file :copy-structure-clean-t)
         copy-structure-clean   (ths/get-shape file :copy-structure-clean)
         copy-structure-clean-t (ths/get-shape file :copy-structure-clean-t)
@@ -1046,6 +1147,8 @@
 
 
         ;; ==== Action: Switch all the copies
+
+
         file' (-> file
                   (tho/swap-component copy-structure-clean :c02 {:new-shape-label :copy-structure-clean-2 :keep-touched? true})
                   (tho/swap-component copy-structure-unif :c02 {:new-shape-label :copy-structure-unif-2 :keep-touched? true})
@@ -1090,6 +1193,8 @@
     ;; Second line:
     ;;   * font size 25
     ;;   * text "new line 2"
+
+
     (t/is (= (get-in copy-structure-unif-t font-size-path-0) "25"))
     (t/is (= (get-in copy-structure-unif-t text-path-0) "new line 1"))
     (t/is (= (get-in copy-structure-unif-t font-size-path-1) "25"))
@@ -1111,6 +1216,8 @@
     ;; Before the switch, second line:
     ;;   * font size 40
     ;;   * text "new line 2"
+
+
     (t/is (= (get-in copy-structure-mixed-t font-size-path-0) "35"))
     (t/is (= (get-in copy-structure-mixed-t text-path-0) "new line 1"))
     (t/is (= (get-in copy-structure-mixed-t font-size-path-1) "40"))
@@ -1124,7 +1231,6 @@
     (t/is (= (get-in copy-structure-mixed-t' text-path-0) "bye"))
     (t/is (nil? (get-in copy-structure-mixed-t' font-size-path-1)))))
 
-
 (t/deftest test-switch-variant-for-other-with-same-nested-component
   (let [;; ==== Setup
         file      (-> (thf/sample-file :file1)
@@ -1144,6 +1250,8 @@
 
 
         ;; On :copy-cp01, change the width of the rect
+
+
         changes (cls/generate-update-shapes (pcb/empty-changes nil (:id page))
                                             #{copy-cp01-rect-id}
                                             (fn [shape]
@@ -1166,8 +1274,6 @@
     ;; The width of copy-cp02-rect' is 25 (change is preserved)
     (t/is (= (:width copy-cp02-rect') 25))))
 
-
-
 (t/deftest test-switch-variant-that-has-swaped-copy
   (let [;; ==== Setup
         file      (-> (thf/sample-file :file1)
@@ -1193,7 +1299,6 @@
         ;; Switch :c01 for :c02
         file'     (tho/swap-component file copy01 :c02 {:new-shape-label :copy02 :keep-touched? true})
 
-
         copy02'    (ths/get-shape file' :copy02)
         copy-cp02' (ths/get-shape file' :copy-cp02)]
     (thf/dump-file file')
@@ -1207,7 +1312,6 @@
     ;;copy-02' had copy-cp02' as child
     (t/is (= (-> copy02' :shapes first) (:id copy-cp02')))))
 
-
 (t/deftest test-switch-variant-that-has-swaped-copy-with-changed-attr
   (let [;; ==== Setup
         file      (-> (thf/sample-file :file1)
@@ -1244,7 +1348,6 @@
         ;; Switch :c01 for :c02
         file'     (tho/swap-component file copy01 :c02 {:new-shape-label :copy02 :keep-touched? true})
 
-
         copy02'    (ths/get-shape file' :copy02)
         copy-cp02' (ths/get-shape file' :copy-cp02)
         copy-cp02-rect' (ths/get-shape-by-id file' (-> copy-cp02' :shapes first))]
@@ -1262,3 +1365,58 @@
     (t/is (= (-> copy02' :shapes first) (:id copy-cp02')))
     ;; The width of copy-cp02-rect' is 25 (change is preserved)
     (t/is (= (:width copy-cp02-rect') 25))))
+
+(t/deftest test-switch-variant-without-touched-but-touched-parent
+  (let [;; ==== Setup
+        file      (-> (thf/sample-file :file1)
+                      (thv/add-variant-with-child
+                       :v01 :c01 :m01 :c02 :m02 :r01 :r02
+                       {:child1-params {:width 5}
+                        :child2-params  {:width 5}})
+                      (tho/add-simple-component :external01 :external01-root :external01-child)
+
+                      (thc/instantiate-component :c01
+                                                 :c01-in-root
+                                                 :children-labels [:r01-in-c01-in-root]
+                                                 :parent-label :external01-root))
+
+        ;; Make a change on r01-in-c01-in-root so it is touched
+        page               (thf/current-page file)
+        r01-in-c01-in-root (ths/get-shape file :r01-in-c01-in-root)
+
+        changes            (cls/generate-update-shapes (pcb/empty-changes nil (:id page))
+                                                       #{(:id r01-in-c01-in-root)}
+                                                       (fn [shape]
+                                                         (assoc shape :width 25))
+                                                       (:objects page)
+                                                       {})
+
+        file               (thf/apply-changes file changes)
+
+
+        ;; Instantiate the component :external01
+
+
+        file        (thc/instantiate-component file
+                                               :external01
+                                               :external-copy01
+                                               :children-labels [:external-copy01-rect :c01-in-copy])
+        page        (thf/current-page file)
+        c01-in-copy (ths/get-shape file :c01-in-copy)
+        rect01      (get-in page [:objects (-> c01-in-copy :shapes first)])
+
+
+        ;; ==== Action
+
+
+        file'        (tho/swap-component file c01-in-copy :c02 {:new-shape-label :c02-in-copy :keep-touched? true})
+
+        page'        (thf/current-page file')
+        c02-in-copy' (ths/get-shape file' :c02-in-copy)
+        rect02'      (get-in page' [:objects (-> c02-in-copy' :shapes first)])]
+
+    (thf/dump-file file :keys [:width :touched])
+    ;; The rect had width 25 before the switch
+    (t/is (= (:width rect01) 25))
+    ;; The rect still has width 25 after the switch
+    (t/is (= (:width rect02') 25))))