mirror/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-12-23 22:48:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag '2.12.0-RC3'

Browse Source
This commit is contained in:
Andrey Antukh
2025-12-12 12:19:29 +01:00
parent 935728aa39 2f46cbc0d4
commit 5abc1aafb4
740 changed files with 64296 additions and 48308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
.circleci/config.yml
View File

@@ -114,7 +114,7 @@ jobs:
# uses the same cache as this task so we prepopulate it
command: |
yarn install
yarn run playwright install chromium
yarn run playwright install chromium --with-deps
- run:
name: "lint scss on frontend"
@@ -207,51 +207,6 @@ jobs:
"npx http-server storybook-static --port 6006 --silent" \
"npx wait-on tcp:6006 && yarn test:storybook"
test-integration:
docker:
- image: penpotapp/devenv:latest
working_directory: ~/repo
resource_class: large
environment:
JAVA_OPTS: -Xmx6g -Xms2g
NODE_OPTIONS: --max-old-space-size=4096
steps:
- checkout
# Download and cache dependencies
- restore_cache:
keys:
- v1-dependencies-{{ checksum "frontend/deps.edn"}}-{{ checksum "frontend/yarn.lock" }}
# Build frontend
- run:
name: "frontend build"
working_directory: "./frontend"
command: |
yarn install
yarn run build:app:assets
yarn run build:app
yarn run build:app:libs
# Build the wasm bundle
- run:
name: "wasm build"
working_directory: "./render-wasm"
command: |
EMSDK_QUIET=1 . /opt/emsdk/emsdk_env.sh
./build release
# Run integration tests
- run:
name: "integration tests"
working_directory: "./frontend"
command: |
yarn run playwright install chromium
yarn run test:e2e -x --workers=4
test-backend:
docker:
- image: penpotapp/devenv:latest
@@ -347,5 +302,4 @@ workflows:
- lint: success
- lint
- test-integration
- test-render-wasm

8
.github/workflows/build-bundle.yml vendored
View File

@@ -57,6 +57,7 @@ jobs:
id: vars
run: |
echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT
- name: Build bundle
env:
@@ -76,14 +77,17 @@ jobs:
- name: Upload Penpot bundle to S3
run: |
aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip
aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip --metadata bundle-version=${{ steps.vars.outputs.bundle_version }}
- name: Notify Mattermost
if: failure()
uses: mattermost/action-mattermost-notify@master
with:
MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
MATTERMOST_CHANNEL: bot-alerts-cicd
TEXT: |
❌ *[PENPOT] Error during the execution of the job*
❌ 📦 *[PENPOT] Error building penpot bundles.*
📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
Bundle version: `${{ steps.vars.outputs.bundle_version }}`
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
@infra

24
.github/workflows/build-docker.yml vendored
View File

@@ -34,12 +34,19 @@ jobs:
echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
- name: Download Penpot Bundles
id: bundles
env:
FILE_NAME: penpot-${{ steps.vars.outputs.gh_ref }}.zip
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
run: |
tmp=$(aws s3api head-object \
--bucket ${{ secrets.S3_BUCKET }} \
--key "$FILE_NAME" \
--query 'Metadata."bundle-version"' \
--output text)
echo "bundle_version=$tmp" >> $GITHUB_OUTPUT
pushd docker/images
aws s3 cp s3://${{ secrets.S3_BUCKET }}/$FILE_NAME .
unzip $FILE_NAME > /dev/null
@@ -59,6 +66,18 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v5
with:
images:
frontend
backend
exporter
storybook
labels: |
bundle_version=${{ steps.bundles.outputs.bundle_version }}
- name: Build and push Backend Docker image
uses: docker/build-push-action@v6
env:
@@ -70,6 +89,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
@@ -84,6 +104,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
@@ -98,6 +119,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
@@ -112,6 +134,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.vars.outputs.gh_ref }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE }}:buildcache,mode=max
@@ -124,5 +147,6 @@ jobs:
TEXT: |
❌ 🐳 *[PENPOT] Error building penpot docker images.*
📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
📦 Bundle: `${{ steps.bundles.outputs.bundle_version }}`
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
@infra

2
.github/workflows/build-tag.yml vendored
View File

@@ -11,7 +11,7 @@ jobs:
secrets: inherit
with:
gh_ref: ${{ github.ref_name }}
build_wasm: "no"
build_wasm: "yes"
build_storybook: "yes"
build-docker:

2
.github/workflows/commit-checker.yml vendored
View File

@@ -26,7 +26,7 @@ jobs:
- name: Check Commit Type
uses: gsactions/commit-message-checker@v2
with:
pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s["A-Z].*[^.]$'
pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert).+[^.])$'
flags: 'gm'
error: 'Commit should match CONTRIBUTING.md guideline'
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

4
.github/workflows/release.yml vendored
View File

@@ -68,12 +68,12 @@ jobs:
for image in "${IMAGES[@]}"; do
skopeo copy --all \
docker://$DOCKER_REGISTRY/$image:$TAG \
docker://docker.io/$PUB_DOCKER_USERNAME/$image:$TAG
docker://docker.io/penpotapp/$image:$TAG
for alias in main latest; do
skopeo copy --all \
docker://$DOCKER_REGISTRY/$image:$TAG \
docker://docker.io/$PUB_DOCKER_USERNAME/$image:$alias
docker://docker.io/penpotapp/$image:$alias
done
done

298
.github/workflows/tests.yml vendored Normal file
View File

@@ -0,0 +1,298 @@
name: "CI"
defaults:
run:
shell: bash
on:
pull_request:
types:
- opened
- synchronize
push:
branches:
- develop
- staging
concurrency:
group: ${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
lint:
name: "Linter"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check clojure code format
run: |
./scripts/lint
test-common:
name: "Common Tests"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Run tests on JVM
working-directory: ./common
run: |
clojure -M:dev:test
- name: Run tests on NODE
working-directory: ./common
run: |
./scripts/test
test-frontend:
name: "Frontend Tests"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Unit Tests
working-directory: ./frontend
run: |
./scripts/test
- name: Component Tests
working-directory: ./frontend
run: |
./scripts/test-components
test-render-wasm:
name: "Render WASM Tests"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Format
working-directory: ./render-wasm
run: |
cargo fmt --check
- name: Lint
working-directory: ./render-wasm
run: |
./lint
- name: Test
working-directory: ./render-wasm
run: |
./test
test-backend:
name: "Backend Tests"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
services:
postgres:
image: postgres:17
# Provide the password for postgres
env:
POSTGRES_USER: penpot_test
POSTGRES_PASSWORD: penpot_test
POSTGRES_DB: penpot_test
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
redis:
image: valkey/valkey:9
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Run tests
working-directory: ./backend
env:
PENPOT_TEST_DATABASE_URI: "postgresql://postgres/penpot_test"
PENPOT_TEST_DATABASE_USERNAME: penpot_test
PENPOT_TEST_DATABASE_PASSWORD: penpot_test
PENPOT_TEST_REDIS_URI: "redis://redis/1"
run: |
clojure -M:dev:test --reporter kaocha.report/documentation
test-library:
name: "Library Tests"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Run tests
working-directory: ./library
run: |
./scripts/test
build-integration:
name: "Build Integration Bundle"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Build Bundle
working-directory: ./frontend
run: |
corepack enable;
corepack install;
yarn install
yarn run build:app:assets
yarn run build:app
yarn run build:app:libs
- name: Build WASM
working-directory: "./render-wasm"
run: |
./build release
- name: Store Bundle Cache
uses: actions/cache@v4
with:
key: "integration-bundle-${{ github.sha }}"
path: frontend/resources/public
test-integration-1:
name: "Integration Tests 1/4"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
needs: build-integration
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Restore Cache
uses: actions/cache/restore@v4
with:
key: "integration-bundle-${{ github.sha }}"
path: frontend/resources/public
- name: Run Tests
working-directory: ./frontend
run: |
./scripts/test-e2e --shard="1/4";
- name: Upload test result
uses: actions/upload-artifact@v4
if: always()
with:
name: integration-tests-result-1
path: frontend/test-results/
overwrite: true
retention-days: 3
test-integration-2:
name: "Integration Tests 2/4"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
needs: build-integration
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Restore Cache
uses: actions/cache/restore@v4
with:
key: "integration-bundle-${{ github.sha }}"
path: frontend/resources/public
- name: Run Tests
working-directory: ./frontend
run: |
./scripts/test-e2e --shard="2/4";
- name: Upload test result
uses: actions/upload-artifact@v4
if: always()
with:
name: integration-tests-result-2
path: frontend/test-results/
overwrite: true
retention-days: 3
test-integration-3:
name: "Integration Tests 3/4"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
needs: build-integration
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Restore Cache
uses: actions/cache/restore@v4
with:
key: "integration-bundle-${{ github.sha }}"
path: frontend/resources/public
- name: Run Tests
working-directory: ./frontend
run: |
./scripts/test-e2e --shard="3/4";
- name: Upload test result
uses: actions/upload-artifact@v4
if: always()
with:
name: integration-tests-result-3
path: frontend/test-results/
overwrite: true
retention-days: 3
test-integration-4:
name: "Integration Tests 4/4"
runs-on: ubuntu-24.04
container: penpotapp/devenv:latest
needs: build-integration
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Restore Cache
uses: actions/cache/restore@v4
with:
key: "integration-bundle-${{ github.sha }}"
path: frontend/resources/public
- name: Run Tests
working-directory: ./frontend
run: |
./scripts/test-e2e --shard="4/4";
- name: Upload test result
uses: actions/upload-artifact@v4
if: always()
with:
name: integration-tests-result-4
path: frontend/test-results/
overwrite: true
retention-days: 3

1
.gitignore vendored
View File

@@ -80,3 +80,4 @@ node_modules
/playwright/.cache/
/render-wasm/target/
/**/.yarn/*
/.pnpm-store

99
CHANGES.md
View File

@@ -1,5 +1,104 @@
# CHANGELOG
## 2.12.0 (Unreleased)
### :boom: Breaking changes & Deprecations
#### Backend RPC API changes
The backend RPC API URLS are changed from `/api/rpc/command/<name>` to
`/api/main/methods/<name>`. The previous PATH is preserved for backward
compatibility; however, if you are a user of this API, it is strongly
recommended that you adapt your code to use the new PATH.
#### Updated SSO Callback URL
The OAuth / Single Sign-On (SSO) callback endpoint has changed to
align with the new OpenID Connect (OIDC) implementation.
Old callback URL:
```
https://<your_domain>/api/auth/oauth/<oauth_provider>/callback
```
New callback URL:
```
https://<your_domain>/api/auth/oidc/callback
```
**Action required:**
If you have SSO/Social-Auth configured on your on-premise instance,
the following actions are required before update:
Update your OAuth or SSO provider configuration (e.g., Okta, Google,
Azure AD, etc.) to use the new callback URL. Failure to update may
result in authentication failures after upgrading.
**Reason for change:**
This update standardizes all authentication flows under the single URL
and makis it more modular, enabling the ability to configure SSO auth
provider dinamically.
#### Changes on default docker compose
We have updated the `docker/images/docker-compose.yaml` with a small
change related to the `PENPOT_SECRET_KEY`. Since this version, this
environment variable is also required on exporter. So if you are using
penpot on-premise you will need to apply the same changes on your own
`docker-compose.yaml` file.
We have removed the Minio server from the `docker/images/docker-compose.yml`
example. It's still usable as before, we just removed the example.
### :rocket: Epics and highlights
### :heart: Community contributions (Thank you!)
- Ensure consistent snap behavior across all zoom levels [Github #7774](https://github.com/penpot/penpot/pull/7774) by [@Tokytome](https://github.com/Tokytome)
- Fix crash in token grid view due to tooltip validation (by @dfelinto) [Github #7887](https://github.com/penpot/penpot/pull/7887)
### :sparkles: New features & Enhancements
- Add the ability to select boards to export as PDF [Taiga #12320](https://tree.taiga.io/project/penpot/issue/12320)
- Add toggle for switching boolean property values [Taiga #12341](https://tree.taiga.io/project/penpot/us/12341)
- Make the file export process more reliable [Taiga #12555](https://tree.taiga.io/project/penpot/us/12555)
- Add auth flow changes [Taiga #12333](https://tree.taiga.io/project/penpot/us/12333)
- Add new shape validation mechanism for shapes [Github #7696](https://github.com/penpot/penpot/pull/7696)
- Apply color tokens from sidebar [Taiga #11353](https://tree.taiga.io/project/penpot/us/11353)
- Display tokens in the inspect tab [Taiga #9313](https://tree.taiga.io/project/penpot/us/9313)
- Refactor clipboard behavior to assess some minor inconsistencies and make pasting binary data faster. [Taiga #12571](https://tree.taiga.io/project/penpot/task/12571)
### :bug: Bugs fixed
- Fix text line-height values are wrong [Taiga #12252](https://tree.taiga.io/project/penpot/issue/12252)
- Fix an error translation [Taiga #12402](https://tree.taiga.io/project/penpot/issue/12402)
- Fix pan cursor not disabling viewport guides [Github #6985](https://github.com/penpot/penpot/issues/6985)
- Fix viewport resize on locked shapes [Taiga #11974](https://tree.taiga.io/project/penpot/issue/11974)
- Fix nested variant in a component doesn't keep inherited overrides [Taiga #12299](https://tree.taiga.io/project/penpot/issue/12299)
- Fix on copy instance inside a components chain touched are missing [Taiga #12371](https://tree.taiga.io/project/penpot/issue/12371)
- Fix problem with multiple selection and shadows [Github #7437](https://github.com/penpot/penpot/issues/7437)
- Fix search shortcut [Taiga #10265](https://tree.taiga.io/project/penpot/issue/10265)
- Fix shortcut conflict in text editor (increase/decrease font size vs word selection)
- Fix problem with plugins generating code for pages different than current one [Taiga #12312](https://tree.taiga.io/project/penpot/issue/12312)
- Fix input confirmation behavior is not uniform [Taiga #12294](https://tree.taiga.io/project/penpot/issue/12294)
- Fix copy/pasting application/transit+json [Taiga #12721](https://tree.taiga.io/project/penpot/issue/12721)
- Fix problem with plugins content attribute [Plugins #209](https://github.com/penpot/penpot-plugins/issues/209)
- Fix U and E icon displayed in project list [Taiga #12806](https://tree.taiga.io/project/penpot/issue/12806)
- Fix unpublish library modal not scrolling a long file list [Taiga #12285](https://tree.taiga.io/project/penpot/issue/12285)
- Fix incorrect interaction betwen hower and scroll on assets sidebar [Taiga #12389](https://tree.taiga.io/project/penpot/issue/12389)
- Fix switch variants with paths [Taiga #12841](https://tree.taiga.io/project/penpot/issue/12841)
- Fix referencing typography tokens on font-family tokens [Taiga #12492](https://tree.taiga.io/project/penpot/issue/12492)
## 2.11.1
- Fix WEBP shape export on docker images [Taiga #3838](https://tree.taiga.io/project/penpot/issue/3838)
## 2.11.0
### :boom: Breaking changes & Deprecations

1
backend/dev/user.clj
View File

@@ -27,6 +27,7 @@
[app.common.transit :as t]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
[app.main :as main]

25
backend/resources/app/email/feedback/en.html
View File

@@ -8,38 +8,41 @@
<body>
<p>
<strong>Feedback from:</strong><br />
{% if profile %}
<span>
<span>Name: </span>
<span><code>{{profile.fullname|abbreviate:25}}</code></span>
</span>
<br />
<span>
<span>Email: </span>
<span>{{profile.email}}</span>
</span>
<br />
<span>
<span>ID: </span>
<span><code>{{profile.id}}</code></span>
</span>
{% else %}
<span>
<span>Email: </span>
<span>{{profile.email}}</span>
</span>
{% endif %}
</p>
<p>
<strong>Subject:</strong><br />
<span>{{subject|abbreviate:300}}</span>
<span>{{feedback-subject|abbreviate:300}}</span>
</p>
<p>
<strong>Type:</strong><br />
<span>{{feedback-type|abbreviate:300}}</span>
</p>
{% if feedback-error-href %}
<p>
<strong>Error HREF:</strong><br />
<span>{{feedback-error-href|abbreviate:500}}</span>
</p>
{% endif %}
<p>
<strong>Message:</strong><br />
{{content|linebreaks-br|safe}}
{{feedback-content|linebreaks-br}}
</p>
</body>
</html>

2
backend/resources/app/email/feedback/en.subj
View File

@@ -1 +1 @@
[PENPOT FEEDBACK]: {{subject}}
[PENPOT FEEDBACK]: {{feedback-subject}}

16
backend/resources/app/email/feedback/en.txt
View File

@@ -1,9 +1,11 @@
{% if profile %}
Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
{% else %}
Feedback from: {{email}}
{% endif %}
From: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
Subject: {{feedback-subject}}
Type: {{feedback-type}}
Subject: {{subject}}
{% if feedback-error-href %}
HREF: {{feedback-error-href}}
{% endif -%}
{{content}}
Message:
{{feedback-content}}

2
backend/resources/app/onboarding.edn
View File

@@ -3,7 +3,7 @@
:file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Tokens%20starter%20kit.penpot"}
{:id "penpot-design-system"
:name "Penpot Design System | Pencil"
:file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/penpot-app.penpot"}
:file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Pencil-Penpot-Design-System.penpot"}
{:id "wireframing-kit"
:name "Wireframe library"
:file-uri "https://github.com/penpot/penpot-files/raw/refs/heads/main/Wireframing%20kit%20v1.1.penpot"}

23
backend/resources/app/templates/api-doc.tmpl
View File

@@ -4,7 +4,7 @@
<meta charset="utf-8" />
<meta name="robots" content="noindex,nofollow">
<meta http-equiv="x-ua-compatible" content="ie=edge" />
<title>Builtin API Documentation - Penpot</title>
<title>{{label|upper}} API Documentation</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
@@ -19,7 +19,7 @@
<body>
<main>
<header>
<h1>Penpot API Documentation (v{{version}})</h1>
<h1>{{label|upper}}: API Documentation (v{{version}})</h1>
<small class="menu">
[
<nav>
@@ -31,9 +31,10 @@
</header>
<section class="doc-content">
<h2>INTRODUCTION</h2>
<p>This documentation is intended to be a general overview of the penpot RPC API.
If you prefer, you can use <a href="/api/openapi.json">OpenAPI</a>
and/or <a href="/api/openapi">SwaggerUI</a> as alternative.</p>
<p>This documentation is intended to be a general overview of
the {{label}} API. If you prefer, you can
use <a href="{{openapi}}">Swagger/OpenAPI</a> as
alternative.</p>
<h2>GENERAL NOTES</h2>
@@ -43,7 +44,7 @@
that starts with <b>get-</b> in the name, can use GET HTTP
method which in many cases benefits from the HTTP cache.</p>
{% block auth-section %}
<h3>Authentication</h3>
<p>The penpot backend right now offers two way for authenticate the request:
<b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
@@ -56,9 +57,10 @@
<p>The access token can be obtained on the appropriate section on profile settings
and it should be provided using <b>`Authorization`</b> header with <b>`Token
&lt;token-string&gt;`</b> value.</p>
{% endblock %}
<h3>Content Negotiation</h3>
<p>The penpot API by default operates indistinctly with: <b>`application/json`</b>
<p>This API operates indistinctly with: <b>`application/json`</b>
and <b>`application/transit+json`</b> content types. You should specify the
desired content-type on the <b>`Accept`</b> header, the transit encoding is used
by default.</p>
@@ -75,13 +77,16 @@
standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
API</a></p>
{% block limits-section %}
<h3>Limits</h3>
<p>The rate limit work per user basis (this means that different api keys share
the same rate limit). For now the limits are not documented because we are
studying and analyzing the data. As a general rule, it should not be abused, if an
abusive use is detected, we will proceed to block the user's access to the
API.</p>
{% endblock %}
{% block webhooks-section %}
<h3>Webhooks</h3>
<p>All methods that emit webhook events are marked with flag <b>WEBHOOK</b>, the
data structure defined on each method represents the <i>payload</i> of the
@@ -97,9 +102,11 @@
"profileId": "db601c95-045f-808b-8002-361312e63531"
}
</pre>
{% endblock %}
</section>
<section class="rpc-doc-content">
<h2>RPC METHODS REFERENCE:</h2>
<h2>METHODS REFERENCE:</h2>
<ul class="rpc-items">
{% for item in methods %}
{% include "app/templates/api-doc-entry.tmpl" with item=item %}

1
backend/resources/app/templates/main-api-doc.tmpl Normal file
View File

@@ -0,0 +1 @@
{% extends "app/templates/api-doc.tmpl" %}

10
backend/resources/app/templates/management-api-doc.tmpl Normal file
View File

@@ -0,0 +1,10 @@
{% extends "app/templates/api-doc.tmpl" %}
{% block auth-section %}
{% endblock %}
{% block limits-section %}
{% endblock %}
{% block webhooks-section %}
{% endblock %}

4
backend/resources/app/templates/openapi.tmpl
View File

@@ -7,7 +7,7 @@
name="description"
content="SwaggerUI"
/>
<title>PENPOT Swagger UI</title>
<title>{{label|upper}} API</title>
<style>{{swagger-css|safe}}</style>
</head>
<body>
@@ -16,7 +16,7 @@
<script>
window.onload = () => {
window.ui = SwaggerUIBundle({
url: '{{public-uri}}/api/openapi.json',
url: '{{uri}}',
dom_id: '#swagger-ui',
presets: [
SwaggerUIBundle.presets.apis,

3
backend/resources/log4j2-devenv-repl.xml
View File

@@ -25,8 +25,7 @@
<Logger name="app.storage.tmp" level="info" />
<Logger name="app.worker" level="trace" />
<Logger name="app.msgbus" level="info" />
<Logger name="app.http.websocket" level="info" />
<Logger name="app.http.sse" level="info" />
<Logger name="app.http" level="info" />
<Logger name="app.util.websocket" level="info" />
<Logger name="app.redis" level="info" />
<Logger name="app.rpc.rlimit" level="info" />

3
backend/resources/log4j2-devenv.xml
View File

@@ -25,8 +25,7 @@
<Logger name="app.storage.tmp" level="info" />
<Logger name="app.worker" level="trace" />
<Logger name="app.msgbus" level="info" />
<Logger name="app.http.websocket" level="info" />
<Logger name="app.http.sse" level="info" />
<Logger name="app.http" level="info" />
<Logger name="app.util.websocket" level="info" />
<Logger name="app.redis" level="info" />
<Logger name="app.rpc.rlimit" level="info" />

17
backend/scripts/_env
View File

@@ -1,18 +1,18 @@
#!/usr/bin/env bash
export PENPOT_MANAGEMENT_API_SHARED_KEY=super-secret-management-api-key
export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key
export PENPOT_SECRET_KEY=super-secret-devenv-key
export PENPOT_HOST=devenv
export PENPOT_PUBLIC_URI=https://localhost:3449
export PENPOT_FLAGS="\
$PENPOT_FLAGS \
enable-login-with-ldap \
enable-login-with-password
enable-login-with-oidc \
enable-login-with-google \
enable-login-with-github \
enable-login-with-gitlab \
disable-login-with-ldap \
disable-login-with-oidc \
disable-login-with-google \
disable-login-with-github \
disable-login-with-gitlab \
enable-backend-worker \
enable-backend-asserts \
disable-feature-fdata-pointer-map \
@@ -20,6 +20,7 @@ export PENPOT_FLAGS="\
enable-audit-log \
enable-transit-readable-response \
enable-demo-users \
enable-user-feedback \
disable-secure-session-cookies \
enable-smtp \
enable-prepl-server \
@@ -46,6 +47,8 @@ export PENPOT_MEDIA_MAX_FILE_SIZE=104857600
# Setup default multipart upload size to 300MiB
export PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE=314572800
export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com"
export AWS_ACCESS_KEY_ID=penpot-devenv
export AWS_SECRET_ACCESS_KEY=penpot-devenv
export PENPOT_OBJECTS_STORAGE_BACKEND=s3

713
backend/src/app/auth/oidc.clj
View File

@@ -15,6 +15,7 @@
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.email.blacklist :as email.blacklist]
@@ -23,7 +24,6 @@
[app.http.errors :as errors]
[app.http.session :as session]
[app.loggers.audit :as audit]
[app.rpc :as rpc]
[app.rpc.commands.profile :as profile]
[app.setup :as-alias setup]
[app.tokens :as tokens]
@@ -34,7 +34,6 @@
[clojure.set :as set]
[cuerdas.core :as str]
[integrant.core :as ig]
[yetti.request :as yreq]
[yetti.response :as-alias yres]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -53,61 +52,53 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- discover-oidc-config
[cfg {:keys [base-uri] :as opts}]
(let [uri (dm/str (u/join base-uri ".well-known/openid-configuration"))
rsp (http/req! cfg {:method :get :uri uri} {:sync? true})]
[cfg {:keys [base-uri] :as provider}]
(let [uri (u/join base-uri ".well-known/openid-configuration")
rsp (http/req! cfg {:method :get :uri (dm/str uri)})]
(if (= 200 (:status rsp))
(let [data (-> rsp :body json/decode)
token-uri (get data :token_endpoint)
auth-uri (get data :authorization_endpoint)
user-uri (get data :userinfo_endpoint)
jwks-uri (get data :jwks_uri)]
jwks-uri (get data :jwks_uri)
logout-uri (get data :end_session_endpoint)]
(l/debug :hint "oidc uris discovered"
:token-uri token-uri
:auth-uri auth-uri
:user-uri user-uri
:jwks-uri jwks-uri)
(-> provider
(assoc :token-uri token-uri)
(assoc :auth-uri auth-uri)
(assoc :user-uri user-uri)
(assoc :jwks-uri jwks-uri)
(assoc :logout-uri logout-uri)))
{:token-uri token-uri
:auth-uri auth-uri
:user-uri user-uri
:jwks-uri jwks-uri})
(do
(l/warn :hint "unable to discover OIDC configuration"
(ex/raise :type ::internal
:code :invalid-sso-config
:hint "unable to discover OIDC configuration"
:discover-uri uri
:http-status (:status rsp))
nil))))
:response-status-code (:status rsp)))))
(defn- prepare-oidc-opts
[cfg]
(let [opts {:base-uri (cf/get :oidc-base-uri)
(def ^:private default-oidc-scopes
#{"openid" "profile" "email"})
(defn- get-oidc-config
"Get the OIDC config params from global config"
[]
(d/without-nils
{:base-uri (cf/get :oidc-base-uri)
:client-id (cf/get :oidc-client-id)
:client-secret (cf/get :oidc-client-secret)
:token-uri (cf/get :oidc-token-uri)
:auth-uri (cf/get :oidc-auth-uri)
:user-uri (cf/get :oidc-user-uri)
:jwks-uri (cf/get :oidc-jwks-uri)
:scopes (cf/get :oidc-scopes #{"openid" "profile" "email"})
:roles-attr (cf/get :oidc-roles-attr)
:scopes (cf/get :oidc-scopes default-oidc-scopes)
:roles (cf/get :oidc-roles)
:name "oidc"}
opts (d/without-nils opts)]
(when (and (string? (:base-uri opts))
(string? (:client-id opts))
(string? (:client-secret opts)))
(if (and (string? (:token-uri opts))
(string? (:user-uri opts))
(string? (:auth-uri opts)))
opts
(try
(-> (discover-oidc-config cfg opts)
(merge opts {:discover? true}))
(catch Throwable cause
(l/warn :hint "unable to discover OIDC configuration"
:cause cause)))))))
:user-info-source (cf/get :oidc-user-info-source)
:roles-attr (cf/get :oidc-roles-attr)
:email-attr (cf/get :oidc-email-attr "email")
:name-attr (cf/get :oidc-name-attr "name")
:type "oidc"
:id "oidc"}))
(defn- process-oidc-jwks
[keys]
@@ -124,20 +115,54 @@
keys))
(defn- fetch-oidc-jwks
[cfg {:keys [jwks-uri]}]
(when jwks-uri
(try
(let [{:keys [status body]} (http/req! cfg {:method :get :uri jwks-uri} {:sync? true})]
[cfg jwks-uri]
(let [{:keys [status body]} (http/req! cfg {:method :get :uri jwks-uri})]
(if (= 200 status)
(-> body json/decode :keys process-oidc-jwks)
(do
(l/warn :hint "unable to retrieve JWKs (unexpected response status code)"
:response-status status
:response-body body)
nil)))
(ex/raise :type ::internal
:code :unable-to-fetch-sso-jwks
:hint "unable to retrieve JWKs (unexpected response status code)"
:response-status-code status))))
(defn- populate-jwks
"Fetch and Add (if possible) JWK's to the OIDC provider"
[cfg provider]
(try
(if-let [jwks (some->> (:jwks-uri provider) (fetch-oidc-jwks cfg))]
(assoc provider :jwks jwks)
provider)
(catch Throwable cause
(l/warn :hint "unable to retrieve JWKs (unexpected exception)"
:cause cause)))))
(l/warn :hint "unable to fetch JWKs for the OIDC provider"
:provider (str (:id provider))
:cause cause)
provider)))
(defn- prepare-oidc-provider
[cfg params]
(when-not (and (string? (:base-uri params))
(string? (:client-id params))
(string? (:client-secret params)))
(ex/raise :type ::internal
:code :invalid-sso-config
:hint "missing params for provider initialization"
:provider (:id params)))
(try
(if (and (string? (:token-uri params))
(string? (:user-uri params))
(string? (:auth-uri params)))
(populate-jwks cfg params)
(let [provider (->> params
(discover-oidc-config cfg)
(populate-jwks cfg))]
(with-meta provider {::discovered true})))
(catch Throwable cause
(ex/raise :type ::internal
:type :invalid-sso-config
:hint "unexpected exception on configuring provider"
:provider (:id params)
:cause cause))))
(defmethod ig/assert-key ::providers/generic
[_ params]
@@ -146,52 +171,64 @@
(defmethod ig/init-key ::providers/generic
[_ cfg]
(when (contains? cf/flags :login-with-oidc)
(if-let [opts (prepare-oidc-opts cfg)]
(let [jwks (fetch-oidc-jwks cfg opts)]
(try
(let [provider (->> (get-oidc-config)
(prepare-oidc-provider cfg))]
(l/inf :hint "provider initialized"
:provider (:id provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider)))
provider)
(catch Throwable cause
(l/warn :hint "unable to initialize auth provider"
:provider "oidc"
:method (if (:discover? opts) "discover" "manual")
:client-id (:client-id opts)
:client-secret (obfuscate-string (:client-secret opts))
:scopes (str/join "," (:scopes opts))
:auth-uri (:auth-uri opts)
:user-uri (:user-uri opts)
:token-uri (:token-uri opts)
:roles-attr (:roles-attr opts)
:roles (:roles opts)
:keys (str/join "," (map str (keys jwks))))
(assoc opts :jwks jwks))
(do
(l/warn :hint "unable to initialize auth provider, missing configuration" :provider "oidc")
nil))))
:cause cause)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; GOOGLE AUTH PROVIDER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/init-key ::providers/google
[_ _]
(let [opts {:client-id (cf/get :google-client-id)
(defn- get-google-config
[]
(d/without-nils
{:client-id (cf/get :google-client-id)
:client-secret (cf/get :google-client-secret)
:scopes #{"openid" "email" "profile"}
:auth-uri "https://accounts.google.com/o/oauth2/v2/auth"
:token-uri "https://oauth2.googleapis.com/token"
:user-uri "https://openidconnect.googleapis.com/v1/userinfo"
:name "google"}]
:user-info-source "userinfo"
:type "google"
:id "google"}))
(defn- prepare-google-provider
[params]
(when-not (and (string? (:client-id params))
(string? (:client-secret params)))
(ex/raise :type ::internal
:code :invalid-sso-config
:hint "missing params for provider initialization"
:provider (:id params)))
params)
(defmethod ig/init-key ::providers/google
[_ _]
(when (contains? cf/flags :login-with-google)
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(do
(try
(let [provider (->> (get-google-config)
(prepare-google-provider))]
(l/inf :hint "provider initialized"
:provider "google"
:client-id (:client-id opts)
:client-secret (obfuscate-string (:client-secret opts)))
opts)
:provider (:id provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider)))
provider)
(do
(l/warn :hint "unable to initialize auth provider, missing configuration" :provider "google")
nil)))))
(catch Throwable cause
(l/warn :hint "unable to initialize auth provider"
:provider "google"
:cause cause)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; GITHUB AUTH PROVIDER
@@ -201,7 +238,7 @@
[val start end]
(and (<= start val) (< val end)))
(defn- retrieve-github-email
(defn- lookup-github-email
[cfg tdata props]
(or (some-> props :github/email)
(let [params {:uri "https://api.github.com/user/emails"
@@ -209,7 +246,7 @@
:timeout 6000
:method :get}
{:keys [status body]} (http/req! cfg params {:sync? true})]
{:keys [status body]} (http/req! cfg params)]
(when-not (int-in-range? status 200 300)
(ex/raise :type :internal
@@ -221,44 +258,61 @@
(->> body json/decode (filter :primary) first :email))))
(defn- get-github-config
[cfg]
(d/without-nils
{:client-id (cf/get :github-client-id)
:client-secret (cf/get :github-client-secret)
:scopes #{"read:user" "user:email"}
:auth-uri "https://github.com/login/oauth/authorize"
:token-uri "https://github.com/login/oauth/access_token"
:user-uri "https://api.github.com/user"
:type "github"
:id "github"
:user-info-source "userinfo"
;; Additional hooks for provider specific way of
;; retrieve emails.
::get-email-fn (partial lookup-github-email cfg)}))
(defn- prepare-github-provider
[params]
(when-not (and (string? (:client-id params))
(string? (:client-secret params)))
(ex/raise :type ::internal
:code :invalid-sso-config
:hint "several required params for configuring GITHUB SSO are missing"
:provider (:id params)))
params)
(defmethod ig/assert-key ::providers/github
[_ params]
(assert (http/client? (::http/client params)) "expected a valid http client"))
(defmethod ig/init-key ::providers/github
[_ cfg]
(let [opts {:client-id (cf/get :github-client-id)
:client-secret (cf/get :github-client-secret)
:scopes #{"read:user" "user:email"}
:auth-uri "https://github.com/login/oauth/authorize"
:token-uri "https://github.com/login/oauth/access_token"
:user-uri "https://api.github.com/user"
:name "github"
;; Additional hooks for provider specific way of
;; retrieve emails.
:get-email-fn (partial retrieve-github-email cfg)}]
(when (contains? cf/flags :login-with-github)
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(do
(try
(let [provider (->> (get-github-config cfg)
(prepare-github-provider))]
(l/inf :hint "provider initialized"
:provider "github"
:client-id (:client-id opts)
:client-secret (obfuscate-string (:client-secret opts)))
opts)
:provider (:id provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider)))
provider)
(do
(l/warn :hint "unable to initialize auth provider, missing configuration" :provider "github")
nil)))))
(catch Throwable cause
(l/warn :hint "unable to initialize auth provider"
:provider "github"
:cause cause)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; GITLAB AUTH PROVIDER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/init-key ::providers/gitlab
[_ cfg]
(defn- get-gitlab-config
[]
(let [base (cf/get :gitlab-base-uri "https://gitlab.com")
opts {:base-uri base
:client-id (cf/get :gitlab-client-id)
@@ -268,21 +322,89 @@
:token-uri (str base "/oauth/token")
:user-uri (str base "/oauth/userinfo")
:jwks-uri (str base "/oauth/discovery/keys")
:name "gitlab"}]
(when (contains? cf/flags :login-with-gitlab)
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(let [jwks (fetch-oidc-jwks cfg opts)]
:type "gitlab"
:id "gitlab"}]
(d/without-nils opts)))
(defn- prepare-gitlab-provider
[cfg params]
(when-not (and (string? (:client-id params))
(string? (:client-secret params)))
(ex/raise :type ::internal
:code :invalid-sso-config
:hint "missing params for provider initialization"
:provider (:id params)))
(try
(let [provider (populate-jwks cfg params)]
(l/inf :hint "provider initialized"
:provider "gitlab"
:base-uri base
:client-id (:client-id opts)
:client-secret (obfuscate-string (:client-secret opts)))
(assoc opts :jwks jwks))
:base-uri (:base-uri provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider)))
provider)
(catch Throwable cause
(ex/raise :type ::internal
:type :invalid-sso-config
:hint "unexpected exception on configuring provider"
:provider (:id params)
:cause cause))))
(do
(l/warn :hint "unable to initialize auth provider, missing configuration" :provider "gitlab")
nil)))))
(defmethod ig/init-key ::providers/gitlab
[_ cfg]
(when (contains? cf/flags :login-with-gitlab)
(try
(let [provider (->> (get-gitlab-config)
(prepare-gitlab-provider cfg))]
(l/inf :hint "provider initialized"
:provider (:id provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider)))
provider)
(catch Throwable cause
(l/warn :hint "unable to initialize auth provider"
:provider "gitlab"
:cause cause)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PROVIDERS COLLECTOR
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def ^:private schema:provider
[:map {:title "Provider"}
[:type ::sm/text]
[:client-id ::sm/text]
[:client-secret ::sm/text]
[:id [:or :string ::sm/uuid]]
[:base-uri {:optional true} ::sm/text]
[:token-uri {:optional true} ::sm/text]
[:auth-uri {:optional true} ::sm/text]
[:user-uri {:optional true} ::sm/text]
[:scopes {:optional true}
[::sm/set ::sm/text]]
[:roles {:optional true}
[::sm/set ::sm/text]]
[:roles-attr {:optional true} ::sm/text]
[:email-attr {:optional true} ::sm/text]
[:name-attr {:optional true} ::sm/text]])
(def ^:private schema:providers
[:map-of :string schema:provider])
(defmethod ig/assert-key ::providers
[_ providers]
(let [check-provider (sm/check-fn schema:provider)]
(assert (every? check-provider (filter identity providers)))))
(defmethod ig/init-key ::providers
[_ providers]
(->> providers
(filter identity)
(d/index-by :id)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HANDLERS
@@ -291,19 +413,19 @@
(defn- parse-attr-path
[provider path]
(let [[fitem & items] (str/split path "__")]
(into [(keyword (:name provider) fitem)] (map keyword) items)))
(into [(keyword (:type provider) fitem)] (map keyword) items)))
(defn- build-redirect-uri
[{:keys [::provider] :as cfg}]
[]
(let [public (u/uri (cf/get :public-uri))]
(str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
(str (assoc public :path (str "/api/auth/oidc/callback")))))
(defn- build-auth-uri
[{:keys [::provider] :as cfg} state]
(defn- build-auth-redirect-uri
[provider token]
(let [params {:client_id (:client-id provider)
:redirect_uri (build-redirect-uri cfg)
:redirect_uri (build-redirect-uri)
:response_type "code"
:state state
:state token
:scope (str/join " " (:scopes provider []))}
query (u/map->query-string params)]
(-> (u/uri (:auth-uri provider))
@@ -312,7 +434,7 @@
(defn- qualify-prop-key
[provider k]
(keyword (:name provider) (name k)))
(keyword (:type provider) (name k)))
(defn- qualify-props
[provider props]
@@ -322,12 +444,12 @@
props))
(defn- fetch-access-token
[{:keys [::provider] :as cfg} code]
[cfg provider code]
(let [params {:client_id (:client-id provider)
:client_secret (:client-secret provider)
:code code
:grant_type "authorization_code"
:redirect_uri (build-redirect-uri cfg)}
:redirect_uri (build-redirect-uri)}
req {:method :post
:headers {"content-type" "application/x-www-form-urlencoded"
"accept" "application/json"}
@@ -335,14 +457,13 @@
:body (u/map->query-string params)}]
(l/trc :hint "fetch access token"
:provider (:name provider)
:provider (:id provider)
:client-id (:client-id provider)
:client-secret (obfuscate-string (:client-secret provider))
:grant-type (:grant_type params)
:redirect-uri (:redirect_uri params))
(let [{:keys [status body]} (http/req! cfg req {:sync? true})]
(l/trc :hint "access token fetched" :status status :body body)
(let [{:keys [status body]} (http/req! cfg req)]
(if (= status 200)
(let [data (json/decode body)
data {:token/access (get data :access_token)
@@ -365,26 +486,30 @@
(letfn [(get-email [props]
;; Allow providers hook into this for custom email
;; retrieval method.
(if-let [get-email-fn (:get-email-fn provider)]
(if-let [get-email-fn (::get-email-fn provider)]
(get-email-fn tdata props)
(let [attr-kw (cf/get :oidc-email-attr "email")
(let [attr-kw (get provider :email-attr "email")
attr-ph (parse-attr-path provider attr-kw)]
(get-in props attr-ph))))
(get-name [props]
(let [attr-kw (cf/get :oidc-name-attr "name")
(or (let [attr-kw (get provider :name-attr "name")
attr-ph (parse-attr-path provider attr-kw)]
(get-in props attr-ph)))]
(get-in props attr-ph))
(let [attr-ph (parse-attr-path provider "nickname")]
(get-in props attr-ph))))]
(let [props (qualify-props provider info)
(let [info (assoc info :provider-id (str (:id provider)))
props (qualify-props provider info)
email (get-email props)]
{:backend (:name provider)
{:backend (:type provider)
:fullname (or (get-name props) email)
:email email
:email-verified (get info :email_verified false)
:props props})))
(defn- fetch-user-info
[{:keys [::provider] :as cfg} tdata]
[cfg provider tdata]
(l/trc :hint "fetch user info"
:uri (:user-uri provider)
:token (obfuscate-string (:token/access tdata)))
@@ -393,7 +518,7 @@
:headers {"Authorization" (str (:token/type tdata) " " (:token/access tdata))}
:timeout 6000
:method :get}
response (http/req! cfg params {:sync? true})]
response (http/req! cfg params)]
(l/trc :hint "user info response"
:status (:status response)
@@ -408,17 +533,16 @@
(-> response :body json/decode)))
(defn- get-user-info
[{:keys [::provider]} tdata]
(defn- get-id-token-claims
[provider tdata]
(try
(when (:token/id tdata)
(let [{:keys [kid alg] :as theader} (jwt/decode-header (:token/id tdata))]
(let [{:keys [kid alg]} (jwt/decode-header (:token/id tdata))]
(when-let [key (if (str/starts-with? (name alg) "hs")
(:client-secret provider)
(get-in provider [:jwks kid]))]
(let [claims (jwt/unsign (:token/id tdata) key {:alg alg})]
(dissoc claims :exp :iss :iat :sid :aud :sub)))))
(jwt/unsign (:token/id tdata) key {:alg alg}))))
(catch Throwable cause
(l/warn :hint "unable to get user info from JWT token (unexpected exception)"
:cause cause))))
@@ -428,41 +552,41 @@
[:backend ::sm/text]
[:email ::sm/email]
[:fullname ::sm/text]
[:props [:map-of :keyword :any]]])
[:email-verified :boolean]
[:props [:map-of :keyword ::sm/any]]])
(def ^:private valid-info?
(sm/validator schema:info))
(defn- get-info
[{:keys [::provider] :as cfg} {:keys [params] :as request}]
(let [state (get params :state)
code (get params :code)
state (tokens/verify cfg {:token state :iss :oauth})
tdata (fetch-access-token cfg code)
info (case (cf/get :oidc-user-info-source)
:token (get-user-info cfg tdata)
:userinfo (fetch-user-info cfg tdata)
(or (get-user-info cfg tdata)
(fetch-user-info cfg tdata)))
[cfg provider state code]
(let [tdata (fetch-access-token cfg provider code)
claims (get-id-token-claims provider tdata)
info (case (get provider :user-info-source)
:token (dissoc claims :exp :iss :iat :aud :sub :sid)
:userinfo (fetch-user-info cfg provider tdata)
(or (some-> claims (dissoc :exp :iss :iat :aud :sub :sid))
(fetch-user-info cfg provider tdata)))
info (process-user-info provider tdata info)]
(l/trc :hint "user info" :info info)
(when-not (valid-info? info)
(l/warn :hint "received incomplete profile info object (please set correct scopes)" :info info)
(if (valid-info? info)
(l/trc :hint "received valid user info object" :info info)
(do
(l/warn :hint "received incomplete user info object (please set correct scopes)" :info info)
(ex/raise :type :internal
:code :incomplete-user-info
:hint "inconmplete user info"
:info info))
:info info)))
;; If the provider is OIDC, we can proceed to check
;; roles if they are defined.
(when (and (= "oidc" (:name provider))
(when (and (= "oidc" (:type provider))
(seq (:roles provider)))
(let [expected-roles (into #{} (:roles provider))
current-roles (let [roles-kw (cf/get :oidc-roles-attr "roles")
current-roles (let [roles-kw (get provider :roles-attr "roles")
roles-ph (parse-attr-path provider roles-kw)
roles (get-in (:props info) roles-ph)]
(cond
@@ -477,6 +601,12 @@
:hint "not enough permissions"))))
(cond-> info
(some? (:sid claims))
(assoc :sso-session-id (:sid claims))
(uuid? (:id provider))
(assoc :sso-provider-id (:id provider))
(some? (:invitation-token state))
(assoc :invitation-token (:invitation-token state))
@@ -489,9 +619,9 @@
(update :props merge (:props state)))))
(defn- get-profile
[cfg info]
[cfg email]
(db/run! cfg (fn [{:keys [::db/conn]}]
(some->> (:email info)
(some->> email
(profile/clean-email)
(profile/get-profile-by-email conn)))))
@@ -511,13 +641,13 @@
(redirect-response uri))))
(defn- redirect-to-register
[cfg info request]
[cfg info provider]
(let [info (assoc info
:iss :prepared-register
:exp (ct/in-future {:hours 48}))
params {:token (tokens/generate cfg info)
:provider (:provider (:path-params request))
:provider (:provider (:id provider))
:fullname (:fullname info)}
params (d/without-nils params)]
@@ -536,43 +666,152 @@
(redirect-response uri)))
(defn- provider-has-email-verified?
[{:keys [::provider] :as cfg} {:keys [props] :as info}]
[provider {:keys [props] :as info}]
(let [prop (qualify-prop-key provider :email_verified)]
(true? (get props prop))))
(defn- profile-has-provider-props?
[{:keys [::provider] :as cfg} profile]
[provider {:keys [props] :as profile}]
(if (and (uuid? (:id provider))
(= "oidc" (:type provider)))
(= (str (:id provider))
(get props :oidc/provider-id))
(let [prop (qualify-prop-key provider :email)]
(contains? (:props profile) prop)))
(contains? props prop))))
(defn- provider-matches-profile?
[{:keys [::provider] :as cfg} profile info]
(or (= (:auth-backend profile) (:name provider))
(profile-has-provider-props? cfg profile)
(provider-has-email-verified? cfg info)))
(defn- decode-row
[{:keys [roles scopes] :as row}]
(cond-> row
(nil? scopes)
(assoc :scopes default-oidc-scopes)
(db/pgarray? scopes)
(assoc :scopes (db/decode-pgarray scopes #{}))
(db/pgarray? roles)
(assoc :roles (db/decode-pgarray roles #{}))))
;; TODO: add cache layer for avoid build an discover each time
(defn get-provider
[cfg id]
(try
(when-let [params (some->> (db/get* cfg :sso-provider {:id id :is-enabled true})
(decode-row))]
(case (:type params)
"oidc" (prepare-oidc-provider cfg params)))
(catch Throwable cause
(l/err :hint "unable to configure custom SSO provider"
:provider (str id)
:cause cause))))
(defn- resolve-provider
[{:keys [::providers] :as cfg} params]
(let [provider (get params :provider)
provider (if (uuid? provider)
provider
(or (uuid/parse* provider) provider))]
(defn- process-callback
[cfg request info profile]
(cond
(some? profile)
(uuid? provider)
(or (get-provider cfg provider)
(ex/raise :type :restriction
:code :sso-provider-not-configured
:hint "provider not configured"
:provider provider))
(string? provider)
(or (get providers provider)
(ex/raise :type :restriction
:code :sso-provider-not-configured
:hint "provider not configured"
:provider provider))
:else
(throw (IllegalArgumentException. "invalid data for provider")))))
(defn- update-profile-with-info
[cfg {:keys [id props] :as profile} info]
(let [props' (merge props (:props info))]
(if (not= props props')
(do
(db/update! cfg :profile
{:props (db/tjson props')}
{:id id}
{::db/return-keys false})
(assoc profile :props props'))
profile)))
(defn- auth-handler
[cfg {:keys [params] :as request}]
(let [provider (resolve-provider cfg params)
props (audit/extract-utm-params params)
esid (audit/get-external-session-id request)
params {:iss "oidc"
:provider (:id provider)
:invitation-token (:invitation-token params)
:external-session-id esid
:props props
:exp (ct/in-future "4h")}
state (tokens/generate cfg (d/without-nils params))
uri (build-auth-redirect-uri provider state)]
{::yres/status 200
::yres/body {:redirect-uri uri}}))
(defn- callback-handler
[cfg {:keys [params] :as request}]
(if-let [error (get params :error)]
(redirect-with-error "unable-to-auth" error)
(try
(let [code (get params :code)
state (get params :state)
state (tokens/verify cfg {:token state :iss "oidc"})
provider (resolve-provider cfg state)
info (get-info cfg provider state code)
profile (get-profile cfg (:email info))]
(cond
(not profile)
(cond
(and (email.blacklist/enabled? cfg)
(email.blacklist/contains? cfg (:email info)))
(redirect-with-error "email-domain-not-allowed")
(and (email.whitelist/enabled? cfg)
(not (email.whitelist/contains? cfg (:email info))))
(redirect-with-error "email-domain-not-allowed")
:else
(if (or (contains? cf/flags :registration)
(contains? cf/flags :oidc-registration))
(redirect-to-register cfg info provider)
(redirect-with-error "registration-disabled")))
(:is-blocked profile)
(redirect-with-error "profile-blocked")
(not (provider-matches-profile? cfg profile info))
(not (or (= (:auth-backend profile) (:type provider))
(profile-has-provider-props? provider profile)
(provider-has-email-verified? provider info)))
(redirect-with-error "auth-provider-not-allowed")
(not (:is-active profile))
(let [info (assoc info :profile-id (:id profile))]
(redirect-to-register cfg info request))
(redirect-to-register cfg info provider))
:else
(let [sxf (session/create-fn cfg (:id profile))
(let [sxf (session/create-fn cfg profile info)
token (or (:invitation-token info)
(tokens/generate cfg
{:iss :auth
:exp (ct/in-future "15m")
:profile-id (:id profile)}))
;; If proceed, update profile on the database
profile (update-profile-with-info cfg profile info)
props (audit/profile->props profile)
context (d/without-nils {:external-session-id (:external-session-id info)})]
@@ -584,96 +823,12 @@
::audit/context context})
(->> (redirect-to-verify-token token)
(sxf request))))
(sxf request)))))
(and (email.blacklist/enabled? cfg)
(email.blacklist/contains? cfg (:email info)))
(redirect-with-error "email-domain-not-allowed")
(and (email.whitelist/enabled? cfg)
(not (email.whitelist/contains? cfg (:email info))))
(redirect-with-error "email-domain-not-allowed")
:else
(let [info (assoc info :is-active (provider-has-email-verified? cfg info))]
(if (or (contains? cf/flags :registration)
(contains? cf/flags :oidc-registration))
(redirect-to-register cfg info request)
(redirect-with-error "registration-disabled")))))
(defn- get-external-session-id
[request]
(let [session-id (yreq/get-header request "x-external-session-id")]
(when (string? session-id)
(if (or (> (count session-id) 256)
(= session-id "null")
(str/blank? session-id))
nil
session-id))))
(defn- auth-handler
[cfg {:keys [params] :as request}]
(let [props (audit/extract-utm-params params)
esid (rpc/get-external-session-id request)
params {:iss :oauth
:invitation-token (:invitation-token params)
:external-session-id esid
:props props
:exp (ct/in-future "4h")}
state (tokens/generate cfg (d/without-nils params))
uri (build-auth-uri cfg state)]
{::yres/status 200
::yres/body {:redirect-uri uri}}))
(defn- callback-handler
[{:keys [::provider] :as cfg} request]
(try
(if-let [error (dm/get-in request [:params :error])]
(redirect-with-error "unable-to-auth" error)
(let [info (get-info cfg request)
profile (get-profile cfg info)]
(process-callback cfg request info profile)))
(catch Throwable cause
(binding [l/*context* (-> (errors/request->context request)
(assoc :auth/provider (:name provider)))]
(let [edata (ex-data cause)]
(cond
(= :validation (:type edata))
(l/wrn :hint "invalid token received" :cause cause)
:else
(l/err :hint "error on oauth process" :cause cause))))
(redirect-with-error "unable-to-auth" (ex-message cause)))))
(def provider-lookup
{:compile
(fn [& _]
(fn [handler {:keys [::providers] :as cfg}]
(fn [request]
(let [provider (some-> request :path-params :provider keyword)]
(if-let [provider (get providers provider)]
(handler (assoc cfg ::provider provider) request)
(ex/raise :type :restriction
:code :provider-not-configured
:provider provider
:hint "provider not configured"))))))})
(def ^:private schema:provider
[:map {:title "provider"}
[:client-id ::sm/text]
[:client-secret ::sm/text]
[:base-uri {:optional true} ::sm/text]
[:token-uri {:optional true} ::sm/text]
[:auth-uri {:optional true} ::sm/text]
[:user-uri {:optional true} ::sm/text]
[:scopes {:optional true}
[::sm/set ::sm/text]]
[:roles {:optional true}
[::sm/set ::sm/text]]
[:roles-attr {:optional true} ::sm/text]
[:email-attr {:optional true} ::sm/text]
[:name-attr {:optional true} ::sm/text]])
(binding [l/*context* (errors/request->context request)]
(l/err :hint "error on process oidc callback" :cause cause)
(redirect-with-error "unable-to-auth" (ex-message cause)))))))
(def ^:private schema:routes-params
[:map
@@ -681,7 +836,7 @@
::http/client
::setup/props
::db/pool
[::providers [:map-of :keyword [:maybe schema:provider]]]])
[::providers schema:providers]])
(defmethod ig/assert-key ::routes
[_ params]
@@ -689,13 +844,11 @@
(defmethod ig/init-key ::routes
[_ cfg]
(let [cfg (update cfg :providers d/without-nils)]
["" {:middleware [[session/authz cfg]
[provider-lookup cfg]]}
["/auth/oauth"
["/:provider"
{:handler auth-handler
(let [cfg (update cfg ::providers d/without-nils)]
["/api/auth/oidc" {:middleware [[session/authz cfg]]}
[""
{:handler (partial auth-handler cfg)
:allowed-methods #{:post}}]
["/:provider/callback"
{:handler callback-handler
:allowed-methods #{:get}}]]]))
["/callback"
{:handler (partial callback-handler cfg)
:allowed-methods #{:get}}]]))

4
backend/src/app/binfile/v3.clj
View File

@@ -255,6 +255,8 @@
(write-entry! output path params)
(events/tap :progress {:section :storage-object :id id})
(with-open [input (sto/get-object-data storage sobject)]
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
(io/copy input output :size (:size sobject))
@@ -279,6 +281,8 @@
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
(events/tap :progress {:section :file :id file-id})
(vswap! bfc/*state* update :files assoc file-id
{:id file-id
:name (:name file)

10
backend/src/app/config.clj
View File

@@ -5,7 +5,6 @@
;; Copyright (c) KALEIDOS INC
(ns app.config
"A configuration management."
(:refer-clojure :exclude [get])
(:require
[app.common.data :as d]
@@ -47,6 +46,7 @@
:auto-file-snapshot-timeout "3h"
:public-uri "http://localhost:3449"
:host "localhost"
:tenant "default"
@@ -57,6 +57,8 @@
:objects-storage-backend "fs"
:objects-storage-fs-directory "assets"
:auth-token-cookie-name "auth-token"
:assets-path "/internal/assets/"
:smtp-default-reply-to "Penpot <no-reply@example.com>"
:smtp-default-from "Penpot <no-reply@example.com>"
@@ -90,7 +92,7 @@
[:secret-key {:optional true} :string]
[:tenant {:optional false} :string]
[:public-uri {:optional false} :string]
[:public-uri {:optional false} ::sm/uri]
[:host {:optional false} :string]
[:http-server-port {:optional true} ::sm/int]
@@ -100,7 +102,7 @@
[:http-server-io-threads {:optional true} ::sm/int]
[:http-server-max-worker-threads {:optional true} ::sm/int]
[:management-api-shared-key {:optional true} :string]
[:management-api-key {:optional true} :string]
[:telemetry-uri {:optional true} :string]
[:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
@@ -165,7 +167,7 @@
[:google-client-id {:optional true} :string]
[:google-client-secret {:optional true} :string]
[:oidc-client-id {:optional true} :string]
[:oidc-user-info-source {:optional true} :keyword]
[:oidc-user-info-source {:optional true} [:enum "auto" "userinfo" "token"]]
[:oidc-client-secret {:optional true} :string]
[:oidc-base-uri {:optional true} :string]
[:oidc-token-uri {:optional true} :string]

6
backend/src/app/db.clj
View File

@@ -704,6 +704,12 @@
(and (sql-exception? cause)
(= "40001" (.getSQLState ^java.sql.SQLException cause))))
(defn duplicate-key-error?
[cause]
(and (sql-exception? cause)
(= "23505" (.getSQLState ^java.sql.SQLException cause))))
(extend-protocol jdbc.prepare/SettableParameter
clojure.lang.Keyword
(set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]

124
backend/src/app/email.clj
View File

@@ -7,6 +7,7 @@
(ns app.email
"Main api for send emails."
(:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.logging :as l]
@@ -93,36 +94,44 @@
headers)))
(defn- assign-body
[^MimeMessage mmsg {:keys [body charset] :or {charset "utf-8"}}]
(let [mpart (MimeMultipart. "mixed")]
[^MimeMessage mmsg {:keys [body charset attachments] :or {charset "utf-8"}}]
(let [mixed-mpart (MimeMultipart. "mixed")]
(cond
(string? body)
(let [bpart (MimeBodyPart.)]
(.setContent bpart ^String body (str "text/plain; charset=" charset))
(.addBodyPart mpart bpart))
(vector? body)
(let [mmp (MimeMultipart. "alternative")
mbp (MimeBodyPart.)]
(.addBodyPart mpart mbp)
(.setContent mbp mmp)
(doseq [item body]
(let [mbp (MimeBodyPart.)]
(.setContent mbp
^String (:content item)
^String (str (:type item "text/plain") "; charset=" charset))
(.addBodyPart mmp mbp))))
(let [text-part (MimeBodyPart.)]
(.setText text-part ^String body ^String charset)
(.addBodyPart mixed-mpart text-part))
(map? body)
(let [bpart (MimeBodyPart.)]
(.setContent bpart
^String (:content body)
^String (str (:type body "text/plain") "; charset=" charset))
(.addBodyPart mpart bpart))
(let [content-part (MimeBodyPart.)
alternative-mpart (MimeMultipart. "alternative")]
(when-let [content (get body "text/plain")]
(let [text-part (MimeBodyPart.)]
(.setText text-part ^String content ^String charset)
(.addBodyPart alternative-mpart text-part)))
(when-let [content (get body "text/html")]
(let [html-part (MimeBodyPart.)]
(.setContent html-part ^String content
(str "text/html; charset=" charset))
(.addBodyPart alternative-mpart html-part)))
(.setContent content-part alternative-mpart)
(.addBodyPart mixed-mpart content-part))
:else
(throw (ex-info "Unsupported type" {:body body})))
(.setContent mmsg mpart)
(throw (IllegalArgumentException. "invalid email body provided")))
(doseq [[name content] attachments]
(prn "attachment" name)
(let [attachment-part (MimeBodyPart.)]
(.setFileName attachment-part ^String name)
(.setContent attachment-part ^String content (str "text/plain; charset=" charset))
(.addBodyPart mixed-mpart attachment-part)))
(.setContent mmsg mixed-mpart)
mmsg))
(defn- opts->props
@@ -210,24 +219,26 @@
(ex/raise :type :internal
:code :missing-email-templates))
{:subject subj
:body (into
[{:type "text/plain"
:content text}]
(when html
[{:type "text/html"
:content html}]))}))
:body (d/without-nils
{"text/plain" text
"text/html" html})}))
(def ^:private schema:context
[:map
(def ^:private schema:params
[:map {:title "Email Params"}
[:to [:or ::sm/email [::sm/vec ::sm/email]]]
[:reply-to {:optional true} ::sm/email]
[:from {:optional true} ::sm/email]
[:lang {:optional true} ::sm/text]
[:subject {:optional true} ::sm/text]
[:priority {:optional true} [:enum :high :low]]
[:extra-data {:optional true} ::sm/text]])
[:extra-data {:optional true} ::sm/text]
[:body {:optional true}
[:or :string [:map-of :string :string]]]
[:attachments {:optional true}
[:map-of :string :string]]])
(def ^:private check-context
(sm/check-fn schema:context))
(def ^:private check-params
(sm/check-fn schema:params))
(defn template-factory
[& {:keys [id schema]}]
@@ -235,9 +246,9 @@
(let [check-fn (if schema
(sm/check-fn schema)
(constantly nil))]
(fn [context]
(let [context (-> context check-context check-fn)
email (build-email-template id context)]
(fn [params]
(let [params (-> params check-params check-fn)
email (build-email-template id params)]
(when-not email
(ex/raise :type :internal
:code :email-template-does-not-exists
@@ -245,35 +256,40 @@
:template-id id))
(cond-> (assoc email :id (name id))
(:extra-data context)
(assoc :extra-data (:extra-data context))
(:extra-data params)
(assoc :extra-data (:extra-data params))
(:from context)
(assoc :from (:from context))
(seq (:attachments params))
(assoc :attachments (:attachments params))
(:reply-to context)
(assoc :reply-to (:reply-to context))
(:from params)
(assoc :from (:from params))
(:to context)
(assoc :to (:to context)))))))
(:reply-to params)
(assoc :reply-to (:reply-to params))
(:to params)
(assoc :to (:to params)))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PUBLIC HIGH-LEVEL API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn render
[email-factory context]
(email-factory context))
[email-factory params]
(email-factory params))
(defn send!
"Schedule an already defined email to be sent using asynchronously
using worker task."
[{:keys [::conn ::factory] :as context}]
[{:keys [::conn ::factory] :as params}]
(assert (db/connectable? conn) "expected a valid database connection or pool")
(let [email (if factory
(factory context)
(dissoc context ::conn))]
(factory params)
(-> params
(dissoc params)
(check-params)))]
(wrk/submit! {::wrk/task :sendmail
::wrk/delay 0
::wrk/max-retries 4
@@ -343,8 +359,10 @@
(def ^:private schema:feedback
[:map
[:subject ::sm/text]
[:content ::sm/text]])
[:feedback-subject ::sm/text]
[:feedback-type ::sm/text]
[:feedback-content ::sm/text]
[:profile :map]])
(def user-feedback
"A profile feedback email."

13
backend/src/app/http.clj
View File

@@ -25,7 +25,6 @@
[app.main :as-alias main]
[app.metrics :as mtx]
[app.rpc :as-alias rpc]
[app.rpc.doc :as-alias rpc.doc]
[app.setup :as-alias setup]
[integrant.core :as ig]
[reitit.core :as r]
@@ -149,7 +148,6 @@
[:map
[::ws/routes schema:routes]
[::rpc/routes schema:routes]
[::rpc.doc/routes schema:routes]
[::oidc/routes schema:routes]
[::assets/routes schema:routes]
[::debug/routes schema:routes]
@@ -171,8 +169,9 @@
[sec/sec-fetch-metadata]
[mw/params]
[mw/format-response]
[session/soft-auth cfg]
[actoken/soft-auth cfg]
[mw/auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)
:token (partial actoken/decode-token cfg)}]
[mw/parse-request]
[mw/errors errors/handle]
[mw/restrict-methods]]}
@@ -188,9 +187,5 @@
(::mgmt/routes cfg)]
(::ws/routes cfg)
["/api" {:middleware [[mw/cors]
[sec/client-header-check]]}
(::oidc/routes cfg)
(::rpc.doc/routes cfg)
(::rpc/routes cfg)]]]))
(::rpc/routes cfg)]]))

59
backend/src/app/http/access_token.clj
View File

@@ -9,23 +9,19 @@
[app.common.logging :as l]
[app.config :as cf]
[app.db :as db]
[app.http :as-alias http]
[app.main :as-alias main]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[yetti.request :as yreq]))
[app.tokens :as tokens]))
(def header-re #"(?i)^Token\s+(.*)")
(defn get-token
[request]
(some->> (yreq/get-header request "authorization")
(re-matches header-re)
(second)))
(defn- decode-token
(defn decode-token
[cfg token]
(when token
(tokens/verify cfg {:token token :iss "access-token"})))
(try
(tokens/verify cfg {:token token :iss "access-token"})
(catch Throwable cause
(l/trc :hint "exception on decoding token"
:token token
:cause cause))))
(def sql:get-token-data
"SELECT perms, profile_id, expires_at
@@ -35,47 +31,28 @@
OR (expires_at > now()));")
(defn- get-token-data
[pool token-id]
[pool claims]
(when-not (db/read-only? pool)
(when-let [token-id (get claims :tid)]
(some-> (db/exec-one! pool [sql:get-token-data token-id])
(update :perms db/decode-pgarray #{}))))
(defn- wrap-soft-auth
"Soft Authentication, will be executed synchronously on the undertow
worker thread."
[handler cfg]
(letfn [(handle-request [request]
(try
(let [token (get-token request)
claims (decode-token cfg token)]
(cond-> request
(map? claims)
(assoc ::id (:tid claims))))
(catch Throwable cause
(l/trace :hint "exception on decoding malformed token" :cause cause)
request)))]
(fn [request]
(handler (handle-request request)))))
(update :perms db/decode-pgarray #{})))))
(defn- wrap-authz
"Authorization middleware, will be executed synchronously on vthread."
[handler {:keys [::db/pool]}]
(fn [request]
(let [{:keys [perms profile-id expires-at]} (some->> (::id request) (get-token-data pool))]
(let [{:keys [type claims]} (get request ::http/auth-data)]
(if (= :token type)
(let [{:keys [perms profile-id expires-at]} (some->> claims (get-token-data pool))]
;; FIXME: revisit this, this data looks unused
(handler (cond-> request
(some? perms)
(assoc ::perms perms)
(some? profile-id)
(assoc ::profile-id profile-id)
(some? expires-at)
(assoc ::expires-at expires-at))))))
(assoc ::expires-at expires-at))))
(def soft-auth
{:name ::soft-auth
:compile (fn [& _]
(when (contains? cf/flags :access-tokens)
wrap-soft-auth))})
(handler request)))))
(def authz
{:name ::authz

16
backend/src/app/http/client.clj
View File

@@ -9,8 +9,7 @@
(:require
[app.common.schema :as sm]
[integrant.core :as ig]
[java-http-clj.core :as http]
[promesa.core :as p])
[java-http-clj.core :as http])
(:import
java.net.http.HttpClient))
@@ -29,14 +28,9 @@
(defn send!
([client req] (send! client req {}))
([client req {:keys [response-type sync?] :or {response-type :string sync? false}}]
([client req {:keys [response-type] :or {response-type :string}}]
(assert (client? client) "expected valid http client")
(if sync?
(http/send req {:client client :as response-type})
(try
(http/send-async req {:client client :as response-type})
(catch Throwable cause
(p/rejected cause))))))
(http/send req {:client client :as response-type})))
(defn- resolve-client
[params]
@@ -56,8 +50,8 @@
([cfg-or-client request]
(let [client (resolve-client cfg-or-client)
request (update request :uri str)]
(send! client request {:sync? true})))
(send! client request {})))
([cfg-or-client request options]
(let [client (resolve-client cfg-or-client)
request (update request :uri str)]
(send! client request (merge {:sync? true} options)))))
(send! client request options))))

9
backend/src/app/http/errors.clj
View File

@@ -13,6 +13,7 @@
[app.config :as cf]
[app.http :as-alias http]
[app.http.access-token :as-alias actoken]
[app.http.auth :as-alias auth]
[app.http.session :as-alias session]
[app.util.inet :as inet]
[clojure.spec.alpha :as s]
@@ -22,16 +23,15 @@
(defn request->context
"Extracts error report relevant context data from request."
[request]
(let [claims (-> {}
(into (::session/token-claims request))
(into (::actoken/token-claims request)))]
(let [{:keys [claims] :as auth} (get request ::http/auth-data)]
(-> (cf/logging-context)
(assoc :request/path (:path request))
(assoc :request/method (:method request))
(assoc :request/params (:params request))
(assoc :request/user-agent (yreq/get-header request "user-agent"))
(assoc :request/ip-addr (inet/parse-request request))
(assoc :request/profile-id (:uid claims))
(assoc :request/profile-id (get claims :uid))
(assoc :request/auth-data auth)
(assoc :version/frontend (or (yreq/get-header request "x-frontend-version") "unknown")))))
(defmulti handle-error
@@ -60,7 +60,6 @@
::yres/body data}
(binding [l/*context* (request->context request)]
(l/wrn :hint "restriction error" :cause err)
{::yres/status 400
::yres/body data}))))

26
backend/src/app/http/management.clj
View File

@@ -13,7 +13,7 @@
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.http.access-token :refer [get-token]]
[app.http.middleware :as mw]
[app.main :as-alias main]
[app.rpc.commands.profile :as cmd.profile]
[app.setup :as-alias setup]
@@ -32,20 +32,6 @@
[_ params]
(assert (db/pool? (::db/pool params)) "expect valid database pool"))
(def ^:private auth
{:name ::auth
:compile
(fn [_ _]
(fn [handler shared-key]
(if shared-key
(fn [request]
(let [token (get-token request)]
(if (= token shared-key)
(handler request)
{::yres/status 403})))
(fn [_ _]
{::yres/status 403}))))})
(def ^:private default-system
{:name ::default-system
:compile
@@ -64,8 +50,12 @@
(db/tx-run! cfg handler request)))))})
(defmethod ig/init-key ::routes
[_ cfg]
["" {:middleware [[auth (cf/get :management-api-shared-key)]
[_ {:keys [::setup/props] :as cfg}]
(let [management-key (or (cf/get :management-api-key)
(get props :management-key))]
["" {:middleware [[mw/shared-key-auth management-key]
[default-system cfg]
[transaction]]}
["/authenticate"
@@ -80,7 +70,7 @@
["/update-customer"
{:handler update-customer
:allowed-methods #{:post}
:transaction true}]])
:transaction true}]]))
;; ---- HELPERS

77
backend/src/app/http/middleware.clj
View File

@@ -12,8 +12,11 @@
[app.common.schema :as-alias sm]
[app.common.transit :as t]
[app.config :as cf]
[app.http :as-alias http]
[app.http.errors :as errors]
[app.tokens :as tokens]
[app.util.pointer-map :as pmap]
[buddy.core.codecs :as bc]
[cuerdas.core :as str]
[yetti.adapter :as yt]
[yetti.middleware :as ymw]
@@ -240,3 +243,77 @@
(if (contains? allowed method)
(handler request)
{::yres/status 405}))))))})
(defn- wrap-auth
[handler decoders]
(let [token-re
#"(?i)^(Token|Bearer)\s+(.*)"
get-token-from-authorization
(fn [request]
(when-let [[_ token-type token] (some->> (yreq/get-header request "authorization")
(re-matches token-re))]
(if (= "token" (str/lower token-type))
{:type :token
:token token}
{:type :bearer
:token token})))
get-token-from-cookie
(fn [request]
(let [cname (cf/get :auth-token-cookie-name)
token (some-> (yreq/get-cookie request cname) :value)]
(when-not (str/empty? token)
{:type :cookie
:token token})))
get-token
(some-fn get-token-from-cookie get-token-from-authorization)
process-request
(fn [request]
(if-let [{:keys [type token] :as auth} (get-token request)]
(let [decode-fn (get decoders type)]
(if (or (= type :cookie) (= type :bearer))
(let [metadata (tokens/decode-header token)]
;; NOTE: we only proceed to decode claims on new
;; cookie tokens. The old cookies dont need to be
;; decoded because they use the token string as ID
(if (and (= (:kid metadata) 1)
(= (:ver metadata) 1)
(some? decode-fn))
(assoc request ::http/auth-data (assoc auth
:claims (decode-fn token)
:metadata metadata))
(assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
(if decode-fn
(assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
(assoc request ::http/auth-data auth))))
request))]
(fn [request]
(-> request process-request handler))))
(def auth
{:name ::auth
:compile (constantly wrap-auth)})
(defn- wrap-shared-key-auth
[handler shared-key]
(if shared-key
(let [shared-key (if (string? shared-key)
shared-key
(bc/bytes->b64-str shared-key true))]
(fn [request]
(let [key (yreq/get-header request "x-shared-key")]
(if (= key shared-key)
(handler request)
{::yres/status 403}))))
(fn [_ _]
{::yres/status 403})))
(def shared-key-auth
{:name ::shared-key-auth
:compile (constantly wrap-shared-key-auth)})

290
backend/src/app/http/session.clj
View File

@@ -11,28 +11,24 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.http :as-alias http]
[app.http.auth :as-alias http.auth]
[app.http.session.tasks :as-alias tasks]
[app.main :as-alias main]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[cuerdas.core :as str]
[integrant.core :as ig]
[yetti.request :as yreq]))
[yetti.request :as yreq]
[yetti.response :as yres]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; DEFAULTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; A default cookie name for storing the session.
(def default-auth-token-cookie-name "auth-token")
;; A cookie that we can use to check from other sites of the same
;; domain if a user is authenticated.
(def default-auth-data-cookie-name "auth-data")
;; Default value for cookie max-age
(def default-cookie-max-age (ct/duration {:days 7}))
@@ -44,10 +40,10 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defprotocol ISessionManager
(read [_ key])
(write! [_ key data])
(update! [_ data])
(delete! [_ key]))
(read-session [_ id])
(create-session [_ params])
(update-session [_ session])
(delete-session [_ id]))
(defn manager?
[o]
@@ -62,71 +58,82 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def ^:private schema:params
[:map {:title "session-params"}
[:user-agent ::sm/text]
[:map {:title "SessionParams" :closed true}
[:profile-id ::sm/uuid]
[:created-at ::ct/inst]])
[:user-agent {:optional true} ::sm/text]
[:sso-provider-id {:optional true} ::sm/uuid]
[:sso-session-id {:optional true} :string]])
(def ^:private valid-params?
(sm/validator schema:params))
(defn- prepare-session-params
[params key]
(assert (string? key) "expected key to be a string")
(assert (not (str/blank? key)) "expected key to be not empty")
(assert (valid-params? params) "expected valid params")
{:user-agent (:user-agent params)
:profile-id (:profile-id params)
:created-at (:created-at params)
:updated-at (:created-at params)
:id key})
(defn- database-manager
[pool]
(reify ISessionManager
(read [_ token]
(db/exec-one! pool (sql/select :http-session {:id token})))
(read-session [_ id]
(if (string? id)
;; Backward compatibility
(let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
(-> session
(assoc :modified-at (:updated-at session))
(dissoc :updated-at)))
(db/exec-one! pool (sql/select :http-session-v2 {:id id}))))
(write! [_ key params]
(let [params (-> params
(assoc :created-at (ct/now))
(prepare-session-params key))]
(db/insert! pool :http-session params)
params))
(create-session [_ params]
(assert (valid-params? params) "expect valid session params")
(update! [_ params]
(let [updated-at (ct/now)]
(db/update! pool :http-session
{:updated-at updated-at}
{:id (:id params)})
(assoc params :updated-at updated-at)))
(let [now (ct/now)
params (-> params
(assoc :id (uuid/next))
(assoc :created-at now)
(assoc :modified-at now))]
(db/insert! pool :http-session-v2 params
{::db/return-keys true})))
(delete! [_ token]
(db/delete! pool :http-session {:id token})
(update-session [_ session]
(let [modified-at (ct/now)]
(if (string? (:id session))
(db/insert! pool :http-session-v2
(-> session
(assoc :id (uuid/next))
(assoc :created-at modified-at)
(assoc :modified-at modified-at)))
(db/update! pool :http-session-v2
{:modified-at modified-at}
{:id (:id session)}
{::db/return-keys true}))))
(delete-session [_ id]
(if (string? id)
(db/delete! pool :http-session {:id id} {::db/return-keys false})
(db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
nil)))
(defn inmemory-manager
[]
(let [cache (atom {})]
(reify ISessionManager
(read [_ token]
(get @cache token))
(read-session [_ id]
(get @cache id))
(write! [_ key params]
(let [params (-> params
(assoc :created-at (ct/now))
(prepare-session-params key))]
(swap! cache assoc key params)
params))
(create-session [_ params]
(assert (valid-params? params) "expect valid session params")
(update! [_ params]
(let [updated-at (ct/now)]
(swap! cache update (:id params) assoc :updated-at updated-at)
(assoc params :updated-at updated-at)))
(let [now (ct/now)
session (-> params
(assoc :id (uuid/next))
(assoc :created-at now)
(assoc :modified-at now))]
(swap! cache assoc (:id session) session)
session))
(delete! [_ token]
(swap! cache dissoc token)
(update-session [_ session]
(let [modified-at (ct/now)]
(swap! cache update (:id session) assoc :modified-at modified-at)
(assoc session :modified-at modified-at)))
(delete-session [_ id]
(swap! cache dissoc id)
nil))))
(defmethod ig/assert-key ::manager
@@ -146,103 +153,116 @@
;; MANAGER IMPL
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare ^:private assign-auth-token-cookie)
(declare ^:private clear-auth-token-cookie)
(declare ^:private gen-token)
(declare ^:private assign-session-cookie)
(declare ^:private clear-session-cookie)
(defn- assign-token
[cfg session]
(let [claims {:iss "authentication"
:aud "penpot"
:sid (:id session)
:iat (:modified-at session)
:uid (:profile-id session)
:sso-provider-id (:sso-provider-id session)
:sso-session-id (:sso-session-id session)}
header {:kid 1 :ver 1}
token (tokens/generate cfg claims header)]
(assoc session :token token)))
(defn create-fn
[{:keys [::manager] :as cfg} profile-id]
[{:keys [::manager] :as cfg} {profile-id :id :as profile}
& {:keys [sso-provider-id sso-session-id]}]
(assert (manager? manager) "expected valid session manager")
(assert (uuid? profile-id) "expected valid uuid for profile-id")
(fn [request response]
(let [uagent (yreq/get-header request "user-agent")
params {:profile-id profile-id
:user-agent uagent}
token (gen-token cfg params)
session (write! manager token params)]
(l/trc :hint "create" :profile-id (str profile-id))
(-> response
(assign-auth-token-cookie session)))))
session (->> {:user-agent uagent
:profile-id profile-id
:sso-provider-id sso-provider-id
:sso-session-id sso-session-id}
(d/without-nils)
(create-session manager)
(assign-token cfg))]
(l/trc :hint "create" :id (str (:id session)) :profile-id (str profile-id))
(assign-session-cookie response session))))
(defn delete-fn
[{:keys [::manager]}]
(assert (manager? manager) "expected valid session manager")
(fn [request response]
(let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
cookie (yreq/get-cookie request cname)]
(l/trc :hint "delete" :profile-id (:profile-id request))
(some->> (:value cookie) (delete! manager))
(-> response
(assoc :status 204)
(assoc :body nil)
(clear-auth-token-cookie)))))
(some->> (get request ::id) (delete-session manager))
(clear-session-cookie response)))
(defn- gen-token
[cfg {:keys [profile-id created-at]}]
(tokens/generate cfg {:iss "authentication"
:iat created-at
:uid profile-id}))
(defn- decode-token
(defn decode-token
[cfg token]
(when token
(tokens/verify cfg {:token token :iss "authentication"})))
(try
(tokens/verify cfg {:token token :iss "authentication"})
(catch Throwable cause
(l/trc :hint "exception on decoding token"
:token token
:cause cause))))
(defn- get-token
(defn get-session
[request]
(let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
cookie (some-> (yreq/get-cookie request cname) :value)]
(when-not (str/empty? cookie)
cookie)))
(get request ::session))
(defn- get-session
[manager token]
(some->> token (read manager)))
(defn invalidate-others
[cfg session]
(let [sql "delete from http_session_v2 where profile_id = ? and id != ?"]
(-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
(db/get-update-count))))
(defn- renew-session?
[{:keys [updated-at] :as session}]
(and (ct/inst? updated-at)
(let [elapsed (ct/diff updated-at (ct/now))]
(neg? (compare default-renewal-max-age elapsed)))))
(defn- wrap-soft-auth
[handler {:keys [::manager] :as cfg}]
(assert (manager? manager) "expected valid session manager")
(letfn [(handle-request [request]
(try
(let [token (get-token request)
claims (decode-token cfg token)]
(cond-> request
(map? claims)
(-> (assoc ::token-claims claims)
(assoc ::token token))))
(catch Throwable cause
(l/trc :hint "exception on decoding malformed token" :cause cause)
request)))]
(fn [request]
(handler (handle-request request)))))
[{:keys [id modified-at] :as session}]
(or (string? id)
(and (ct/inst? modified-at)
(let [elapsed (ct/diff modified-at (ct/now))]
(neg? (compare default-renewal-max-age elapsed))))))
(defn- wrap-authz
[handler {:keys [::manager]}]
[handler {:keys [::manager] :as cfg}]
(assert (manager? manager) "expected valid session manager")
(fn [request]
(let [session (get-session manager (::token request))
(let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
(cond
(= type :cookie)
(let [session (case (:ver metadata)
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
0 (read-session manager token)
1 (some->> (:sid claims) (read-session manager))
nil)
request (cond-> request
(some? session)
(assoc ::profile-id (:profile-id session)
::id (:id session)))
(-> (assoc ::profile-id (:profile-id session))
(assoc ::session session)))
response (handler request)]
(if (renew-session? session)
(let [session (update! manager session)]
(-> response
(assign-auth-token-cookie session)))
response))))
(if (and session (renew-session? session))
(let [session (->> session
(update-session manager)
(assign-token cfg))]
(assign-session-cookie response session))
response))
(def soft-auth
{:name ::soft-auth
:compile (constantly wrap-soft-auth)})
(= type :bearer)
(let [session (case (:ver metadata)
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
0 (read-session manager token)
1 (some->> (:sid claims) (read-session manager))
nil)
request (cond-> request
(some? session)
(-> (assoc ::profile-id (:profile-id session))
(assoc ::session session)))]
(handler request))
:else
(handler request)))))
(def authz
{:name ::authz
@@ -250,16 +270,16 @@
;; --- IMPL
(defn- assign-auth-token-cookie
[response {token :id updated-at :updated-at}]
(defn- assign-session-cookie
[response {token :token modified-at :modified-at}]
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
created-at updated-at
created-at modified-at
renewal (ct/plus created-at default-renewal-max-age)
expires (ct/plus created-at max-age)
secure? (contains? cf/flags :secure-session-cookies)
strict? (contains? cf/flags :strict-session-cookies)
cors? (contains? cf/flags :cors)
name (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
name (cf/get :auth-token-cookie-name)
comment (str "Renewal at: " (ct/format-inst renewal :rfc1123))
cookie {:path "/"
:http-only true
@@ -268,12 +288,12 @@
:comment comment
:same-site (if cors? :none (if strict? :strict :lax))
:secure secure?}]
(update response :cookies assoc name cookie)))
(update response ::yres/cookies assoc name cookie)))
(defn- clear-auth-token-cookie
(defn- clear-session-cookie
[response]
(let [cname (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
(update response :cookies assoc cname {:path "/" :value "" :max-age 0})))
(let [cname (cf/get :auth-token-cookie-name)]
(update response ::yres/cookies assoc cname {:path "/" :value "" :max-age 0})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TASK: SESSION GC

79
backend/src/app/loggers/audit.clj
View File

@@ -25,7 +25,8 @@
[app.util.inet :as inet]
[app.util.services :as-alias sv]
[app.worker :as wrk]
[cuerdas.core :as str]))
[cuerdas.core :as str]
[yetti.request :as yreq]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HELPERS
@@ -78,17 +79,32 @@
(remove #(contains? reserved-props (key %))))
props))
(defn event-from-rpc-params
"Create a base event skeleton with pre-filled some important
data that can be extracted from RPC params object"
[params]
(let [context {:external-session-id (::rpc/external-session-id params)
:external-event-origin (::rpc/external-event-origin params)
:triggered-by (::rpc/handler-name params)}]
{::type "action"
::profile-id (::rpc/profile-id params)
::ip-addr (::rpc/ip-addr params)
::context (d/without-nils context)}))
(defn get-external-session-id
[request]
(when-let [session-id (yreq/get-header request "x-external-session-id")]
(when-not (or (> (count session-id) 256)
(= session-id "null")
(str/blank? session-id))
session-id)))
(defn- get-client-event-origin
[request]
(when-let [origin (yreq/get-header request "x-event-origin")]
(when-not (or (= origin "null")
(str/blank? origin))
(str/prune origin 200))))
(defn get-client-user-agent
[request]
(when-let [user-agent (yreq/get-header request "user-agent")]
(str/prune user-agent 500)))
(defn- get-client-version
[request]
(when-let [origin (yreq/get-header request "x-frontend-version")]
(when-not (or (= origin "null")
(str/blank? origin))
(str/prune origin 100))))
;; --- SPECS
@@ -117,6 +133,33 @@
(def ^:private check-event
(sm/check-fn schema:event))
(defn- prepare-context-from-request
[request]
(let [client-event-origin (get-client-event-origin request)
client-version (get-client-version request)
client-user-agent (get-client-user-agent request)
session-id (get-external-session-id request)
token-id (::actoken/id request)]
(d/without-nils
{:external-session-id session-id
:access-token-id (some-> token-id str)
:client-event-origin client-event-origin
:client-user-agent client-user-agent
:client-version client-version
:version (:full cf/version)})))
(defn event-from-rpc-params
"Create a base event skeleton with pre-filled some important
data that can be extracted from RPC params object"
[params]
(let [context (some-> params meta ::http/request prepare-context-from-request)
event {::type "action"
::profile-id (or (::rpc/profile-id params) uuid/zero)
::ip-addr (::rpc/ip-addr params)}]
(cond-> event
(some? context)
(assoc ::context context))))
(defn prepare-event
[cfg mdata params result]
(let [resultm (meta result)
@@ -126,23 +169,15 @@
(::rpc/profile-id params)
uuid/zero)
session-id (get params ::rpc/external-session-id)
event-origin (get params ::rpc/external-event-origin)
props (-> (or (::replace-props resultm)
(-> params
(merge (::props resultm))
(dissoc :profile-id)
(dissoc :type)))
(clean-props))
token-id (::actoken/id request)
context (-> (::context resultm)
(assoc :external-session-id session-id)
(assoc :external-event-origin event-origin)
(assoc :access-token-id (some-> token-id str))
(d/without-nils))
context (merge (::context resultm)
(prepare-context-from-request request))
ip-addr (inet/parse-request request)]
{::type (or (::type resultm)

2
backend/src/app/loggers/audit/archive_task.clj
View File

@@ -57,7 +57,7 @@
:uid uuid/zero})
body (t/encode {:events events})
headers {"content-type" "application/transit+json"
"origin" (cf/get :public-uri)
"origin" (str (cf/get :public-uri))
"cookie" (u/map->query-string {:auth-token token})}
params {:uri uri
:timeout 12000

2
backend/src/app/loggers/database.clj
View File

@@ -49,7 +49,7 @@
ctx (-> context
(assoc :tenant (cf/get :tenant))
(assoc :host (cf/get :host))
(assoc :public-uri (cf/get :public-uri))
(assoc :public-uri (str (cf/get :public-uri)))
(assoc :logger/name logger)
(assoc :logger/level level)
(dissoc :request/params :value :params :data))]

32
backend/src/app/main.clj
View File

@@ -21,7 +21,7 @@
[app.http.client :as-alias http.client]
[app.http.debug :as-alias http.debug]
[app.http.management :as mgmt]
[app.http.session :as-alias session]
[app.http.session :as session]
[app.http.session.tasks :as-alias session.tasks]
[app.http.websocket :as http.ws]
[app.loggers.webhooks :as-alias webhooks]
@@ -31,7 +31,6 @@
[app.redis :as-alias rds]
[app.rpc :as-alias rpc]
[app.rpc.climit :as-alias climit]
[app.rpc.doc :as-alias rpc.doc]
[app.setup :as-alias setup]
[app.srepl :as-alias srepl]
[app.storage :as-alias sto]
@@ -260,14 +259,17 @@
::oidc.providers/generic
{::http.client/client (ig/ref ::http.client/client)}
::oidc/providers
[(ig/ref ::oidc.providers/google)
(ig/ref ::oidc.providers/github)
(ig/ref ::oidc.providers/gitlab)
(ig/ref ::oidc.providers/generic)]
::oidc/routes
{::http.client/client (ig/ref ::http.client/client)
::db/pool (ig/ref ::db/pool)
::setup/props (ig/ref ::setup/props)
::oidc/providers {:google (ig/ref ::oidc.providers/google)
:github (ig/ref ::oidc.providers/github)
:gitlab (ig/ref ::oidc.providers/gitlab)
:oidc (ig/ref ::oidc.providers/generic)}
::oidc/providers (ig/ref ::oidc/providers)
::session/manager (ig/ref ::session/manager)
::email/blacklist (ig/ref ::email/blacklist)
::email/whitelist (ig/ref ::email/whitelist)}
@@ -280,7 +282,6 @@
{::session/manager (ig/ref ::session/manager)
::db/pool (ig/ref ::db/pool)
::rpc/routes (ig/ref ::rpc/routes)
::rpc.doc/routes (ig/ref ::rpc.doc/routes)
::setup/props (ig/ref ::setup/props)
::mtx/routes (ig/ref ::mtx/routes)
::oidc/routes (ig/ref ::oidc/routes)
@@ -300,6 +301,7 @@
{::db/pool (ig/ref ::db/pool)
::mtx/metrics (ig/ref ::mtx/metrics)
::mbus/msgbus (ig/ref ::mbus/msgbus)
::setup/props (ig/ref ::setup/props)
::session/manager (ig/ref ::session/manager)}
:app.http.assets/routes
@@ -337,11 +339,23 @@
::email/blacklist (ig/ref ::email/blacklist)
::email/whitelist (ig/ref ::email/whitelist)}
:app.rpc.doc/routes
{:app.rpc/methods (ig/ref :app.rpc/methods)}
:app.rpc/management-methods
{::http.client/client (ig/ref ::http.client/client)
::db/pool (ig/ref ::db/pool)
::rds/pool (ig/ref ::rds/pool)
::wrk/executor (ig/ref ::wrk/netty-executor)
::session/manager (ig/ref ::session/manager)
::sto/storage (ig/ref ::sto/storage)
::mtx/metrics (ig/ref ::mtx/metrics)
::mbus/msgbus (ig/ref ::mbus/msgbus)
::rds/client (ig/ref ::rds/client)
::setup/props (ig/ref ::setup/props)}
::rpc/routes
{::rpc/methods (ig/ref :app.rpc/methods)
::rpc/management-methods (ig/ref :app.rpc/management-methods)
;; FIXME: revisit if db/pool is necessary here
::db/pool (ig/ref ::db/pool)
::session/manager (ig/ref ::session/manager)
::setup/props (ig/ref ::setup/props)}

55
backend/src/app/media.clj
View File

@@ -17,6 +17,7 @@
[app.common.time :as ct]
[app.config :as cf]
[app.db :as-alias db]
[app.http.client :as http]
[app.storage :as-alias sto]
[app.storage.tmp :as tmp]
[buddy.core.bytes :as bb]
@@ -37,6 +38,9 @@
org.im4java.core.IMOperation
org.im4java.core.Info))
(def default-max-file-size
(* 1024 1024 10)) ; 10 MiB
(def schema:upload
[:map {:title "Upload"}
[:filename :string]
@@ -241,7 +245,7 @@
(ex/raise :type :validation
:code :invalid-svg-file
:hint "uploaded svg does not provides dimensions"))
(merge input info {:ts (ct/now)}))
(merge input info {:ts (ct/now) :size (fs/size path)}))
(let [instance (Info. (str path))
mtype' (.getProperty instance "Mime type")]
@@ -261,6 +265,7 @@
(assoc input
:width width
:height height
:size (fs/size path)
:ts (ct/now)))))))
(defmethod process-error org.im4java.core.InfoException
@@ -270,6 +275,54 @@
:hint "invalid image"
:cause error))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; IMAGE HELPERS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn download-image
"Download an image from the provided URI and return the media input object"
[{:keys [::http/client]} uri]
(letfn [(parse-and-validate [{:keys [headers] :as response}]
(let [size (some-> (get headers "content-length") d/parse-integer)
mtype (get headers "content-type")
format (cm/mtype->format mtype)
max-size (cf/get :media-max-file-size default-max-file-size)]
(when-not size
(ex/raise :type :validation
:code :unknown-size
:hint "seems like the url points to resource with unknown size"))
(when (> size max-size)
(ex/raise :type :validation
:code :file-too-large
:hint (str/ffmt "the file size % is greater than the maximum %"
size
default-max-file-size)))
(when (nil? format)
(ex/raise :type :validation
:code :media-type-not-allowed
:hint "seems like the url points to an invalid media object"))
{:size size :mtype mtype :format format}))]
(let [{:keys [body] :as response} (http/req! client
{:method :get :uri uri}
{:response-type :input-stream})
{:keys [size mtype]} (parse-and-validate response)
path (tmp/tempfile :prefix "penpot.media.download.")
written (io/write* path body :size size)]
(when (not= written size)
(ex/raise :type :internal
:code :mismatch-write-size
:hint "unexpected state: unable to write to file"))
{;; :size size
:path path
:mtype mtype})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FONTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

8
backend/src/app/migrations.clj
View File

@@ -450,7 +450,13 @@
:fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}
{:name "0141-add-file-data-table.sql"
:fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}])
:fn (mg/resource "app/migrations/sql/0141-add-file-data-table.sql")}
{:name "0142-add-sso-provider-table"
:fn (mg/resource "app/migrations/sql/0142-add-sso-provider-table.sql")}
{:name "0143-http-session-v2-table"
:fn (mg/resource "app/migrations/sql/0143-add-http-session-v2-table.sql")}])
(defn apply-migrations!
[pool name migrations]

33
backend/src/app/migrations/sql/0142-add-sso-provider-table.sql Normal file
View File

@@ -0,0 +1,33 @@
CREATE TABLE sso_provider (
id uuid PRIMARY KEY,
created_at timestamptz NOT NULL DEFAULT now(),
modified_at timestamptz NOT NULL DEFAULT now(),
is_enabled boolean NOT NULL DEFAULT true,
type text NOT NULL CHECK (type IN ('oidc')),
domain text NOT NULL,
client_id text NOT NULL,
client_secret text NOT NULL,
base_uri text NOT NULL,
token_uri text NULL,
auth_uri text NULL,
user_uri text NULL,
jwks_uri text NULL,
logout_uri text NULL,
roles_attr text NULL,
email_attr text NULL,
name_attr text NULL,
user_info_source text NOT NULL DEFAULT 'token'
CHECK (user_info_source IN ('token', 'userinfo', 'auto')),
scopes text[] NULL,
roles text[] NULL
);
CREATE UNIQUE INDEX sso_provider__domain__idx
ON sso_provider(domain);

23
backend/src/app/migrations/sql/0143-add-http-session-v2-table.sql Normal file
View File

@@ -0,0 +1,23 @@
CREATE TABLE http_session_v2 (
id uuid PRIMARY KEY,
created_at timestamptz NOT NULL DEFAULT now(),
modified_at timestamptz NOT NULL DEFAULT now(),
profile_id uuid REFERENCES profile(id) ON DELETE CASCADE,
user_agent text NULL,
sso_provider_id uuid NULL REFERENCES sso_provider(id) ON DELETE CASCADE,
sso_session_id text NULL
);
CREATE INDEX http_session_v2__profile_id__idx
ON http_session_v2(profile_id);
CREATE INDEX http_session_v2__sso_provider_id__idx
ON http_session_v2(sso_provider_id)
WHERE sso_provider_id IS NOT NULL;
CREATE INDEX http_session_v2__sso_session_id__idx
ON http_session_v2(sso_session_id)
WHERE sso_session_id IS NOT NULL;

175
backend/src/app/rpc.clj
View File

@@ -13,11 +13,14 @@
[app.common.schema :as sm]
[app.common.spec :as us]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
[app.http :as-alias http]
[app.http.access-token :as actoken]
[app.http.client :as-alias http.client]
[app.http.middleware :as mw]
[app.http.security :as sec]
[app.http.session :as session]
[app.loggers.audit :as audit]
[app.main :as-alias main]
@@ -26,6 +29,7 @@
[app.redis :as rds]
[app.rpc.climit :as climit]
[app.rpc.cond :as cond]
[app.rpc.doc :as doc]
[app.rpc.helpers :as rph]
[app.rpc.retry :as retry]
[app.rpc.rlimit :as rlimit]
@@ -36,7 +40,6 @@
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.core :as p]
[yetti.request :as yreq]
[yetti.response :as yres]))
@@ -44,7 +47,7 @@
(defn- default-handler
[_]
(p/rejected (ex/error :type :not-found)))
(ex/raise :type :not-found))
(defn- handle-response-transformation
[response request mdata]
@@ -65,58 +68,44 @@
response (if (fn? result)
(result request)
(let [result (rph/unwrap result)
status (::http/status mdata 200)
status (or (::http/status mdata)
(if (nil? result)
204
200))
headers (cond-> (::http/headers mdata {})
(yres/stream-body? result)
(assoc "content-type" "application/octet-stream"))]
{::yres/status status
::yres/headers headers
::yres/body result}))]
(-> response
(handle-response-transformation request mdata)
(handle-before-comple-hook mdata))))
(defn get-external-session-id
[request]
(when-let [session-id (yreq/get-header request "x-external-session-id")]
(when-not (or (> (count session-id) 256)
(= session-id "null")
(str/blank? session-id))
session-id)))
(defn- get-external-event-origin
[request]
(when-let [origin (yreq/get-header request "x-event-origin")]
(when-not (or (> (count origin) 256)
(= origin "null")
(str/blank? origin))
origin)))
(defn- rpc-handler
(defn- make-rpc-handler
"Ring handler that dispatches cmd requests and convert between
internal async flow into ring async flow."
[methods {:keys [params path-params method] :as request}]
[methods]
(let [methods (update-vals methods peek)]
(fn [{:keys [params path-params method] :as request}]
(let [handler-name (:type path-params)
etag (yreq/get-header request "if-none-match")
profile-id (or (::session/profile-id request)
(::actoken/profile-id request))
ip-addr (inet/parse-request request)
session-id (get-external-session-id request)
event-origin (get-external-event-origin request)
data (-> params
(assoc ::handler-name handler-name)
(assoc ::ip-addr ip-addr)
(assoc ::request-at (ct/now))
(assoc ::external-session-id session-id)
(assoc ::external-event-origin event-origin)
(assoc ::session/id (::session/id request))
(assoc ::cond/key etag)
(cond-> (uuid? profile-id)
(assoc ::profile-id profile-id)))
data (vary-meta data assoc ::http/request request)
data (with-meta data
{::http/request request})
handler-fn (get methods (keyword handler-name) default-handler)]
(when (and (or (= method :get)
@@ -126,9 +115,10 @@
:code :method-not-allowed
:hint "method not allowed for this request"))
;; FIXME: why we have this cond enabled here, we need to move it outside this handler
(binding [cond/*enabled* true]
(let [response (handler-fn data)]
(handle-response request response)))))
(handle-response request response)))))))
(defn- wrap-metrics
"Wrap service method with metrics measurement."
@@ -205,7 +195,7 @@
::sm/explain (explain params)))))))
f))
(defn- wrap-all
(defn- wrap
[cfg f mdata]
(as-> f $
(wrap-db-transaction cfg $ mdata)
@@ -219,17 +209,30 @@
(wrap-params-validation cfg $ mdata)
(wrap-authentication cfg $ mdata)))
(defn- wrap
(defn- wrap-management
[cfg f mdata]
(l/trc :hint "register method" :name (::sv/name mdata))
(let [f (wrap-all cfg f mdata)]
(partial f cfg)))
(as-> f $
(wrap-db-transaction cfg $ mdata)
(retry/wrap-retry cfg $ mdata)
(climit/wrap cfg $ mdata)
(wrap-metrics cfg $ mdata)
(wrap-audit cfg $ mdata)
(wrap-spec-conform cfg $ mdata)
(wrap-params-validation cfg $ mdata)
(wrap-authentication cfg $ mdata)))
(defn- process-method
[cfg [vfn mdata]]
[(keyword (::sv/name mdata)) [mdata (wrap cfg vfn mdata)]])
[cfg module wrap-fn [f mdata]]
(l/trc :hint "add method" :module module :name (::sv/name mdata))
(let [f (wrap-fn cfg f mdata)
k (keyword (::sv/name mdata))]
[k [mdata (partial f cfg)]]))
(defn- resolve-command-methods
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; API METHODS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- resolve-methods
[cfg]
(let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
(->> (sv/scan-ns
@@ -258,7 +261,7 @@
'app.rpc.commands.verify-token
'app.rpc.commands.viewer
'app.rpc.commands.webhooks)
(map (partial process-method cfg))
(map (partial process-method cfg "rpc" wrap))
(into {}))))
(def ^:private schema:methods-params
@@ -282,7 +285,50 @@
(defmethod ig/init-key ::methods
[_ cfg]
(let [cfg (d/without-nils cfg)]
(resolve-command-methods cfg)))
(resolve-methods cfg)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MANAGEMENT METHODS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- resolve-management-methods
[cfg]
(let [cfg (assoc cfg ::type "management" ::metrics-id :rpc-management-timing)]
(->> (sv/scan-ns
'app.rpc.management.subscription
'app.rpc.management.exporter)
(map (partial process-method cfg "management" wrap-management))
(into {}))))
(def ^:private schema:management-methods-params
[:map {:title "management-methods-params"}
::session/manager
::http.client/client
::db/pool
::rds/pool
::mbus/msgbus
::sto/storage
::mtx/metrics
::setup/props])
(defmethod ig/assert-key ::management-methods
[_ params]
(assert (sm/check schema:management-methods-params params)))
(defmethod ig/init-key ::management-methods
[_ cfg]
(let [cfg (d/without-nils cfg)]
(resolve-management-methods cfg)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ROUTES
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- redirect
[href]
(fn [_]
{::yres/status 308
::yres/headers {"location" (str href)}}))
(def ^:private schema:methods
[:map-of :keyword [:tuple :map ::sm/fn]])
@@ -297,11 +343,50 @@
(assert (db/pool? (::db/pool params)) "expect valid database pool")
(assert (some? (::setup/props params)))
(assert (session/manager? (::session/manager params)) "expect valid session manager")
(assert (valid-methods? (::methods params)) "expect valid methods map"))
(assert (valid-methods? (::methods params)) "expect valid methods map")
(assert (valid-methods? (::management-methods params)) "expect valid methods map"))
(defmethod ig/init-key ::routes
[_ {:keys [::methods] :as cfg}]
(let [methods (update-vals methods peek)]
[["/rpc" {:middleware [[session/authz cfg]
[actoken/authz cfg]]}
["/command/:type" {:handler (partial rpc-handler methods)}]]]))
[_ {:keys [::methods ::management-methods ::setup/props] :as cfg}]
(let [public-uri (cf/get :public-uri)
management-key (or (cf/get :management-api-key)
(get props :management-key))]
["/api"
["/management"
["/methods/:type"
{:middleware [[mw/shared-key-auth management-key]
[session/authz cfg]]
:handler (make-rpc-handler management-methods)}]
(doc/routes :methods management-methods
:label "management"
:base-uri (u/join public-uri "/api/management")
:description "MANAGEMENT API")]
["/main"
["/methods/:type"
{:middleware [[mw/cors]
[sec/client-header-check]
[session/authz cfg]
[actoken/authz cfg]]
:handler (make-rpc-handler methods)}]
(doc/routes :methods methods
:label "main"
:base-uri (u/join public-uri "/api/main")
:description "MAIN API")]
;; BACKWARD COMPATIBILITY
["/_doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
["/doc" {:handler (redirect (u/join public-uri "/api/main/doc"))}]
["/openapi" {:handler (redirect (u/join public-uri "/api/main/doc/openapi"))}]
["/openapi.join" {:handler (redirect (u/join public-uri "/api/main/doc/openapi.json"))}]
["/rpc/command/:type"
{:middleware [[mw/cors]
[sec/client-header-check]
[session/authz cfg]
[actoken/authz cfg]]
:handler (make-rpc-handler methods)}]]))

1
backend/src/app/rpc/commands/access_token.clj
View File

@@ -28,6 +28,7 @@
expires-at (some-> expiration (ct/in-future))
created-at (ct/now)
token (tokens/generate cfg {:iss "access-token"
:uid profile-id
:iat created-at
:tid token-id})

118
backend/src/app/rpc/commands/auth.clj
View File

@@ -7,21 +7,24 @@
(ns app.rpc.commands.auth
(:require
[app.auth :as auth]
[app.auth.oidc :as oidc]
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.email :as eml]
[app.email.blacklist :as email.blacklist]
[app.email.whitelist :as email.whitelist]
[app.http :as-alias http]
[app.http.session :as session]
[app.loggers.audit :as audit]
[app.media :as media]
[app.rpc :as-alias rpc]
[app.rpc.climit :as-alias climit]
[app.rpc.commands.profile :as profile]
@@ -30,6 +33,7 @@
[app.rpc.helpers :as rph]
[app.setup :as-alias setup]
[app.setup.welcome-file :refer [create-welcome-file]]
[app.storage :as sto]
[app.tokens :as tokens]
[app.util.services :as sv]
[app.worker :as wrk]
@@ -109,7 +113,7 @@
(assoc profile :is-admin (let [admins (cf/get :admins)]
(contains? admins (:email profile)))))]
(-> response
(rph/with-transform (session/create-fn cfg (:id profile)))
(rph/with-transform (session/create-fn cfg profile))
(rph/with-meta {::audit/props (audit/profile->props profile)
::audit/profile-id (:id profile)}))))]
@@ -145,7 +149,24 @@
[cfg params]
(if (= (:profile-id params)
(::rpc/profile-id params))
(rph/with-transform {} (session/delete-fn cfg))
(let [{:keys [claims]}
(rph/get-auth-data params)
provider
(some->> (get claims :sso-provider-id)
(oidc/get-provider cfg))
response
(if (and provider (:logout-uri provider))
(let [params {"logout_hint" (get claims :sso-session-id)
"client_id" (get provider :client-id)
"post_logout_redirect_uri" (str (cf/get :public-uri))}
uri (-> (u/uri (:logout-uri provider))
(assoc :query (u/map->query-string params)))]
{:redirect-uri uri})
{})]
(rph/with-transform response (session/delete-fn cfg)))
{}))
;; ---- COMMAND: Recover Profile
@@ -271,11 +292,29 @@
;; ---- COMMAND: Register Profile
(defn create-profile!
(defn import-profile-picture
[cfg uri]
(try
(let [storage (sto/resolve cfg)
input (media/download-image cfg uri)
input (media/run {:cmd :info :input input})
hash (sto/calculate-hash (:path input))
content (-> (sto/content (:path input) (:size input))
(sto/wrap-with-hash hash))
sobject (sto/put-object! storage {::sto/content content
::sto/deduplicate? true
:bucket "profile"
:content-type (:mtype input)})]
(:id sobject))
(catch Throwable cause
(l/err :hint "unable to import profile picture"
:cause cause)
nil)))
(defn create-profile
"Create the profile entry on the database with limited set of input
attrs (all the other attrs are filled with default values)."
[conn {:keys [email] :as params}]
(dm/assert! ::sm/email email)
[{:keys [::db/conn] :as cfg} {:keys [email] :as params}]
(let [id (or (:id params) (uuid/next))
props (-> (audit/extract-utm-params params)
(merge (:props params))
@@ -283,8 +322,7 @@
:viewed-walkthrough? false
:nudge {:big 10 :small 1}
:v2-info-shown true
:release-notes-viewed (:main cf/version)})
(db/tjson))
:release-notes-viewed (:main cf/version)}))
password (or (:password params) "!")
@@ -299,6 +337,12 @@
theme (:theme params nil)
email (str/lower email)
photo-id (some->> (or (:oidc/picture props)
(:google/picture props)
(:github/picture props)
(:gitlab/picture props))
(import-profile-picture cfg))
params {:id id
:fullname (:fullname params)
:email email
@@ -306,27 +350,26 @@
:lang locale
:password password
:deleted-at (:deleted-at params)
:props props
:props (db/tjson props)
:theme theme
:photo-id photo-id
:is-active is-active
:is-muted is-muted
:is-demo is-demo}]
(try
(-> (db/insert! conn :profile params)
(profile/decode-row))
(catch org.postgresql.util.PSQLException cause
(let [state (.getSQLState cause)]
(if (not= state "23505")
(throw cause)
(do
(l/error :hint "not an error" :cause cause)
(if (db/duplicate-key-error? cause)
(ex/raise :type :validation
:code :email-already-exists
:hint "email already exists"
:cause cause))))))))
:cause cause)
(throw cause))))))
(defn create-profile-rels!
(defn create-profile-rels
[conn {:keys [id] :as profile}]
(let [features (cfeat/get-enabled-features cf/flags)
team (teams/create-team conn
@@ -376,12 +419,13 @@
;; to detect if the profile is already registered
(or (profile/get-profile-by-email conn (:email claims))
(let [is-active (or (boolean (:is-active claims))
(boolean (:email-verified claims))
(not (contains? cf/flags :email-verification)))
params (-> params
(assoc :is-active is-active)
(update :password auth/derive-password))
profile (->> (create-profile! conn params)
(create-profile-rels! conn))]
profile (->> (create-profile cfg params)
(create-profile-rels conn))]
(vary-meta profile assoc :created true))))
created? (-> profile meta :created true?)
@@ -419,10 +463,10 @@
(and (some? invitation)
(= (:email profile)
(:member-email invitation)))
(let [claims (assoc invitation :member-id (:id profile))
token (tokens/generate cfg claims)]
(let [invitation (assoc invitation :member-id (:id profile))
token (tokens/generate cfg invitation)]
(-> {:invitation-token token}
(rph/with-transform (session/create-fn cfg (:id profile)))
(rph/with-transform (session/create-fn cfg profile claims))
(rph/with-meta {::audit/replace-props props
::audit/context {:action "accept-invitation"}
::audit/profile-id (:id profile)})))
@@ -433,7 +477,7 @@
created?
(if (:is-active profile)
(-> (profile/strip-private-attrs profile)
(rph/with-transform (session/create-fn cfg (:id profile)))
(rph/with-transform (session/create-fn cfg profile claims))
(rph/with-defer create-welcome-file-when-needed)
(rph/with-meta
{::audit/replace-props props
@@ -562,4 +606,32 @@
[cfg params]
(db/tx-run! cfg request-profile-recovery params))
;; --- COMMAND: get-sso-config
(defn- extract-domain
"Extract the domain part from email"
[email]
(let [at (str/last-index-of email "@")]
(when (and (>= at 0)
(< at (dec (count email))))
(-> (subs email (inc at))
(str/trim)
(str/lower)))))
(def ^:private schema:get-sso-provider
[:map {:title "get-sso-config"}
[:email ::sm/email]])
(def ^:private schema:get-sso-provider-result
[:map {:title "SSOProvider"}
[:id ::sm/uuid]])
(sv/defmethod ::get-sso-provider
{::rpc/auth false
::doc/added "2.12"
::sm/params schema:get-sso-provider
::sm/result schema:get-sso-provider-result}
[cfg {:keys [email]}]
(when-let [domain (extract-domain email)]
(when-let [config (db/get* cfg :sso-provider {:domain domain})]
(select-keys config [:id]))))

58
backend/src/app/rpc/commands/binfile.clj
View File

@@ -11,9 +11,9 @@
[app.binfile.v1 :as bf.v1]
[app.binfile.v3 :as bf.v3]
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
[app.http.sse :as sse]
@@ -25,10 +25,12 @@
[app.rpc.commands.projects :as projects]
[app.rpc.commands.teams :as teams]
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.tasks.file-gc]
[app.util.services :as sv]
[app.worker :as-alias wrk]))
[app.worker :as-alias wrk]
[datoteka.fs :as fs]))
(set! *warn-on-reflection* true)
@@ -38,52 +40,42 @@
schema:export-binfile
[:map {:title "export-binfile"}
[:file-id ::sm/uuid]
[:version {:optional true} ::sm/int]
[:include-libraries ::sm/boolean]
[:embed-assets ::sm/boolean]])
(defn stream-export-v1
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
(rph/stream
(fn [_ output-stream]
(defn- export-binfile
[{:keys [::sto/storage] :as cfg} {:keys [file-id include-libraries embed-assets]}]
(let [output (tmp/tempfile*)]
(try
(-> cfg
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets embed-assets)
(assoc ::bfc/include-libraries include-libraries)
(bf.v1/export-files! output-stream))
(catch Throwable cause
(l/err :hint "exception on exporting file"
:file-id (str file-id)
:cause cause))))))
(bf.v3/export-files! output))
(defn stream-export-v3
[cfg {:keys [file-id include-libraries embed-assets] :as params}]
(rph/stream
(fn [_ output-stream]
(try
(-> cfg
(assoc ::bfc/ids #{file-id})
(assoc ::bfc/embed-assets embed-assets)
(assoc ::bfc/include-libraries include-libraries)
(bf.v3/export-files! output-stream))
(catch Throwable cause
(l/err :hint "exception on exporting file"
:file-id (str file-id)
:cause cause))))))
(let [data (sto/content output)
object (sto/put-object! storage
{::sto/content data
::sto/touched-at (ct/in-future {:minutes 60})
:content-type "application/zip"
:bucket "tempfile"})]
(-> (cf/get :public-uri)
(u/join "/assets/by-id/")
(u/join (str (:id object)))))
(finally
(fs/delete output)))))
(sv/defmethod ::export-binfile
"Export a penpot file in a binary format."
{::doc/added "1.15"
::doc/changes [["2.12" "Remove version parameter, only one version is supported"]]
::webhooks/event? true
::sm/params schema:export-binfile}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id version file-id] :as params}]
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
(files/check-read-permissions! pool profile-id file-id)
(let [version (or version 1)]
(case (int version)
1 (stream-export-v1 cfg params)
2 (throw (ex-info "not-implemented" {}))
3 (stream-export-v3 cfg params))))
(sse/response (partial export-binfile cfg params)))
;; --- Command: import-binfile

8
backend/src/app/rpc/commands/demo.clj
View File

@@ -39,7 +39,7 @@
fullname (str "Demo User " sem)
password (-> (bn/random-bytes 16)
(bc/bytes->b64u)
(bc/bytes->b64 true)
(bc/bytes->str))
params {:email email
@@ -49,9 +49,9 @@
:deleted-at (ct/in-future (cf/get-deletion-delay))
:password (derive-password password)
:props {}}
profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
(->> (auth/create-profile! conn params)
(auth/create-profile-rels! conn))))]
profile (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
(->> (auth/create-profile cfg params)
(auth/create-profile-rels conn))))]
(with-meta {:email email
:password password}
{::audit/profile-id (:id profile)})))

32
backend/src/app/rpc/commands/feedback.clj
View File

@@ -7,6 +7,7 @@
(ns app.rpc.commands.feedback
"A general purpose feedback module."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.config :as cf]
@@ -21,8 +22,11 @@
(def ^:private schema:send-user-feedback
[:map {:title "send-user-feedback"}
[:subject [:string {:max 400}]]
[:content [:string {:max 2500}]]])
[:subject [:string {:max 500}]]
[:content [:string {:max 2500}]]
[:type {:optional true} :string]
[:error-href {:optional true} [:string {:max 2500}]]
[:error-report {:optional true} :string]])
(sv/defmethod ::send-user-feedback
{::doc/added "1.18"
@@ -39,16 +43,26 @@
(defn- send-user-feedback!
[pool profile params]
(let [dest (or (cf/get :user-feedback-destination)
(let [destination
(or (cf/get :user-feedback-destination)
;; LEGACY
(cf/get :feedback-destination))]
(cf/get :feedback-destination))
attachments
(d/without-nils
{"error-report.txt" (:error-report params)})]
(eml/send! {::eml/conn pool
::eml/factory eml/user-feedback
:from dest
:to dest
:profile profile
:from (cf/get :smtp-default-from)
:to destination
:reply-to (:email profile)
:email (:email profile)
:subject (:subject params)
:content (:content params)})
:attachments attachments
:feedback-subject (:subject params)
:feedback-type (:type params "not-specified")
:feedback-content (:content params)
:feedback-error-href (:error-href params)
:profile profile})
nil))

191
backend/src/app/rpc/commands/files.clj
View File

@@ -26,6 +26,7 @@
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata]
[app.features.logical-deletion :as ldel]
[app.http.sse :as sse]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.msgbus :as mbus]
@@ -38,6 +39,7 @@
[app.rpc.helpers :as rph]
[app.rpc.permissions :as perms]
[app.util.blob :as blob]
[app.util.events :as events]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.worker :as wrk]
@@ -353,9 +355,8 @@
::sm/params schema:get-project-files
::sm/result schema:files}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id]}]
(dm/with-open [conn (db/open pool)]
(projects/check-read-permissions! conn profile-id project-id)
(get-project-files conn project-id)))
(projects/check-read-permissions! pool profile-id project-id)
(get-project-files pool project-id))
;; --- COMMAND QUERY: has-file-libraries
@@ -424,7 +425,6 @@
;; --- QUERY COMMAND: get-page
(defn- prune-objects
"Given the page data and the object-id returns the page data with all
other not needed objects removed from the `:objects` data
@@ -765,6 +765,54 @@
(teams/check-read-permissions! conn profile-id team-id)
(get-team-recent-files conn team-id)))
;; --- COMMAND QUERY: get-team-deleted-files
(def sql:team-deleted-files
"WITH deleted_files AS (
SELECT f.id,
f.revn,
f.vern,
f.project_id,
f.created_at,
f.modified_at,
f.name,
f.is_shared,
f.deleted_at AS will_be_deleted_at,
ft.media_id AS thumbnail_id,
row_number() OVER w AS row_num,
p.team_id
FROM file AS f
INNER JOIN project AS p ON (p.id = f.project_id)
LEFT JOIN file_thumbnail AS ft on (ft.file_id = f.id
AND ft.revn = f.revn
AND ft.deleted_at is null)
WHERE p.team_id = ?
AND (p.deleted_at > ?::timestamptz OR
f.deleted_at > ?::timestamptz)
WINDOW w AS (PARTITION BY f.project_id
ORDER BY f.modified_at DESC)
ORDER BY f.modified_at DESC
)
SELECT * FROM deleted_files")
(defn get-team-deleted-files
[conn team-id]
(let [now (ct/now)]
(db/exec! conn [sql:team-deleted-files team-id now now])))
(def ^:private schema:get-team-deleted-files
[:map {:title "get-team-deleted-files"}
[:team-id ::sm/uuid]])
(sv/defmethod ::get-team-deleted-files
{::doc/added "2.12"
::sm/params schema:get-team-deleted-files}
[cfg {:keys [::rpc/profile-id team-id]}]
(db/run! cfg (fn [{:keys [::db/conn]}]
(teams/check-read-permissions! conn profile-id team-id)
(get-team-deleted-files conn team-id))))
;; --- COMMAND QUERY: get-file-info
@@ -1113,3 +1161,138 @@
(check-edition-permissions! conn profile-id file-id)
(-> (ignore-sync conn params)
(update :features db/decode-pgarray #{})))
;; --- MUTATION COMMAND: delete-files-immediatelly
(def ^:private sql:delete-team-files
"UPDATE file AS uf SET deleted_at = ?::timestamptz
FROM (
SELECT f.id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
JOIN team AS t ON (t.id = p.team_id)
WHERE t.deleted_at IS NULL
AND t.id = ?
AND f.id = ANY(?::uuid[])
) AS subquery
WHERE uf.id = subquery.id
RETURNING uf.id, uf.deleted_at;")
(def ^:private schema:permanently-delete-team-files
[:map {:title "permanently-delete-team-files"}
[:team-id ::sm/uuid]
[:ids [::sm/set ::sm/uuid]]])
(sv/defmethod ::permanently-delete-team-files
"Mark the specified files to be deleted immediatelly on the
specified team. The team-id on params will be used to filter and
check writable permissons on team."
{::doc/added "2.12"
::sm/params schema:permanently-delete-team-files
::db/transaction true}
[{:keys [::db/conn]} {:keys [::rpc/profile-id ::rpc/request-at team-id ids]}]
(teams/check-edition-permissions! conn profile-id team-id)
(reduce (fn [acc {:keys [id deleted-at]}]
(wrk/submit! {::db/conn conn
::wrk/task :delete-object
::wrk/params {:object :file
:deleted-at deleted-at
:id id}})
(conj acc id))
#{}
(db/plan conn [sql:delete-team-files request-at team-id
(db/create-array conn "uuid" ids)])))
;; --- MUTATION COMMAND: restore-files-immediatelly
(def ^:private sql:resolve-editable-files
"SELECT f.id, f.project_id
FROM file AS f
JOIN project AS p ON (p.id = f.project_id)
JOIN team AS t ON (t.id = p.team_id)
WHERE t.deleted_at IS NULL
AND t.id = ?
AND f.id = ANY(?::uuid[])")
(defn- restore-file
[conn file-id]
(db/update! conn :file
{:deleted-at nil
:has-media-trimmed false}
{:id file-id}
{::db/return-keys false})
(db/update! conn :file-media-object
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-change
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-data
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-thumbnail
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-tagged-object-thumbnail
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false}))
(def ^:private sql:restore-projects
"UPDATE project SET deleted_at = null WHERE id = ANY(?::uuid[])")
(defn- restore-projects
[conn project-ids]
(let [project-ids (db/create-array conn "uuid" project-ids)]
(->> (db/exec-one! conn [sql:restore-projects project-ids])
(db/get-update-count))))
(defn- restore-deleted-team-files
[{:keys [::db/conn]} {:keys [::rpc/profile-id team-id ids]}]
(teams/check-edition-permissions! conn profile-id team-id)
(let [total-files
(count ids)
{:keys [files projects]}
(reduce (fn [result {:keys [id project-id]}]
(let [index (-> result :files count)]
(events/tap :progress {:file-id id :index index :total total-files})
(restore-file conn id)
(-> result
(update :files conj id)
(update :projects conj project-id))))
{:files #{} :projectes #{}}
(db/plan conn [sql:resolve-editable-files team-id
(db/create-array conn "uuid" ids)]))]
(restore-projects conn projects)
files))
(def ^:private schema:restore-deleted-team-files
[:map {:title "restore-deleted-team-files"}
[:team-id ::sm/uuid]
[:ids [::sm/set ::sm/uuid]]])
(sv/defmethod ::restore-deleted-team-files
"Removes the deletion mark from the specified files (and respective
projects) on the specified team."
{::doc/added "2.12"
::sse/stream? true
::sm/params schema:restore-deleted-team-files}
[cfg params]
(sse/response #(db/tx-run! cfg restore-deleted-team-files params)))

17
backend/src/app/rpc/commands/files_thumbnails.clj
View File

@@ -96,7 +96,7 @@
;; loading all pages into memory for find the frame set for thumbnail.
(defn get-file-data-for-thumbnail
[{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
[{:keys [::db/conn] :as cfg} {:keys [data id] :as file} strip-frames-with-thumbnails]
(letfn [;; function responsible on finding the frame marked to be
;; used as thumbnail; the returned frame always have
;; the :page-id set to the page that it belongs.
@@ -173,7 +173,7 @@
;; Assoc the available thumbnails and prune not visible shapes
;; for avoid transfer unnecessary data.
:always
strip-frames-with-thumbnails
(update :objects assoc-thumbnails page-id thumbs)))))
(def ^:private
@@ -186,7 +186,8 @@
[:map {:title "PartialFile"}
[:id ::sm/uuid]
[:revn {:min 0} ::sm/int]
[:page [:map-of :keyword ::sm/any]]])
[:page [:map-of :keyword ::sm/any]]
[:strip-frames-with-thumbnails {:optional true} ::sm/boolean]])
(sv/defmethod ::get-file-data-for-thumbnail
"Retrieves the data for generate the thumbnail of the file. Used
@@ -195,7 +196,7 @@
::doc/module :files
::sm/params schema:get-file-data-for-thumbnail
::sm/result schema:partial-file}
[cfg {:keys [::rpc/profile-id file-id] :as params}]
[cfg {:keys [::rpc/profile-id file-id strip-frames-with-thumbnails] :as params}]
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
(files/check-read-permissions! conn profile-id file-id)
@@ -205,14 +206,18 @@
file (bfc/get-file cfg file-id
:realize? true
:read-only? true)]
:read-only? true)
strip-frames-with-thumbnails
(or (nil? strip-frames-with-thumbnails) ;; if not present, default to true
(true? strip-frames-with-thumbnails))]
(-> (cfeat/get-team-enabled-features cf/flags team)
(cfeat/check-file-features! (:features file)))
{:file-id file-id
:revn (:revn file)
:page (get-file-data-for-thumbnail cfg file)}))))
:page (get-file-data-for-thumbnail cfg file strip-frames-with-thumbnails)}))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MUTATION COMMANDS

8
backend/src/app/rpc/commands/ldap.clj
View File

@@ -66,12 +66,12 @@
:member-email (:email profile))
token (tokens/generate cfg claims)]
(-> {:invitation-token token}
(rph/with-transform (session/create-fn cfg (:id profile)))
(rph/with-transform (session/create-fn cfg profile))
(rph/with-meta {::audit/props (:props profile)
::audit/profile-id (:id profile)})))
(-> (profile/strip-private-attrs profile)
(rph/with-transform (session/create-fn cfg (:id profile)))
(rph/with-transform (session/create-fn cfg profile))
(rph/with-meta {::audit/props (:props profile)
::audit/profile-id (:id profile)}))))))
@@ -83,6 +83,6 @@
(profile/clean-email)
(profile/get-profile-by-email conn))
(->> (assoc info :is-active true :is-demo false)
(auth/create-profile! conn)
(auth/create-profile-rels! conn)
(auth/create-profile cfg)
(auth/create-profile-rels conn)
(profile/strip-private-attrs))))))

60
backend/src/app/rpc/commands/media.clj
View File

@@ -7,14 +7,10 @@
(ns app.rpc.commands.media
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.media :as cm]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http.client :as http]
[app.loggers.audit :as-alias audit]
[app.media :as media]
[app.rpc :as-alias rpc]
@@ -22,13 +18,7 @@
[app.rpc.commands.files :as files]
[app.rpc.doc :as-alias doc]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.services :as sv]
[cuerdas.core :as str]
[datoteka.io :as io]))
(def default-max-file-size
(* 1024 1024 10)) ; 10 MiB
[app.util.services :as sv]))
(def thumbnail-options
{:width 100
@@ -197,56 +187,12 @@
mobj))
(defn download-image
[{:keys [::http/client]} uri]
(letfn [(parse-and-validate [{:keys [headers] :as response}]
(let [size (some-> (get headers "content-length") d/parse-integer)
mtype (get headers "content-type")
format (cm/mtype->format mtype)
max-size (cf/get :media-max-file-size default-max-file-size)]
(when-not size
(ex/raise :type :validation
:code :unknown-size
:hint "seems like the url points to resource with unknown size"))
(when (> size max-size)
(ex/raise :type :validation
:code :file-too-large
:hint (str/ffmt "the file size % is greater than the maximum %"
size
default-max-file-size)))
(when (nil? format)
(ex/raise :type :validation
:code :media-type-not-allowed
:hint "seems like the url points to an invalid media object"))
{:size size :mtype mtype :format format}))]
(let [{:keys [body] :as response} (http/req! client
{:method :get :uri uri}
{:response-type :input-stream :sync? true})
{:keys [size mtype]} (parse-and-validate response)
path (tmp/tempfile :prefix "penpot.media.download.")
written (io/write* path body :size size)]
(when (not= written size)
(ex/raise :type :internal
:code :mismatch-write-size
:hint "unexpected state: unable to write to file"))
{:filename "tempfile"
:size size
:path path
:mtype mtype})))
(defn- create-file-media-object-from-url
[cfg {:keys [url name] :as params}]
(let [content (download-image cfg url)
(let [content (media/download-image cfg url)
params (-> params
(assoc :content content)
(assoc :name (or name (:filename content))))]
(assoc :name (d/nilv name "unknown")))]
;; NOTE: we use the climit here in a dynamic invocation because we
;; don't want saturate the process-image limit with IO (download

24
backend/src/app/rpc/commands/profile.clj
View File

@@ -154,7 +154,6 @@
(declare validate-password!)
(declare update-profile-password!)
(declare invalidate-profile-session!)
(def ^:private
schema:update-profile-password
@@ -169,8 +168,7 @@
::climit/id :auth/global
::db/transaction true}
[cfg {:keys [::rpc/profile-id password] :as params}]
(let [profile (validate-password! cfg (assoc params :profile-id profile-id))
session-id (::session/id params)]
(let [profile (validate-password! cfg (assoc params :profile-id profile-id))]
(when (= (:email profile) (str/lower (:password params)))
(ex/raise :type :validation
@@ -178,14 +176,12 @@
:hint "you can't use your email as password"))
(update-profile-password! cfg (assoc profile :password password))
(invalidate-profile-session! cfg profile-id session-id)
nil))
(defn- invalidate-profile-session!
"Removes all sessions except the current one."
[{:keys [::db/conn]} profile-id session-id]
(let [sql "delete from http_session where profile_id = ? and id != ?"]
(:next.jdbc/update-count (db/exec-one! conn [sql profile-id session-id]))))
(->> (rph/get-request params)
(session/get-session)
(session/invalidate-others cfg))
nil))
(defn- validate-password!
[{:keys [::db/conn] :as cfg} {:keys [profile-id old-password] :as params}]
@@ -284,9 +280,9 @@
:file-path (str (:path file))
:file-mtype (:mtype file)}}))))
(defn- generate-thumbnail!
[_ file]
(let [input (media/run {:cmd :info :input file})
(defn- generate-thumbnail
[_ input]
(let [input (media/run {:cmd :info :input input})
thumb (media/run {:cmd :profile-thumbnail
:format :jpeg
:quality 85
@@ -307,7 +303,7 @@
(assoc ::climit/id [[:process-image/by-profile (:profile-id params)]
[:process-image/global]])
(assoc ::climit/label "upload-photo")
(climit/invoke! generate-thumbnail! file))]
(climit/invoke! generate-thumbnail file))]
(sto/put-object! storage params)))
;; --- MUTATION: Request Email Change

62
backend/src/app/rpc/commands/projects.clj
View File

@@ -70,7 +70,27 @@
;; --- QUERY: Get projects
(declare get-projects)
(def ^:private sql:projects
"SELECT p.*,
coalesce(tpp.is_pinned, false) as is_pinned,
(SELECT count(*) FROM file AS f
WHERE f.project_id = p.id
AND f.deleted_at is null) AS count,
(SELECT count(*) FROM file AS f
WHERE f.project_id = p.id) AS total_count
FROM project AS p
INNER JOIN team AS t ON (t.id = p.team_id)
LEFT JOIN team_project_profile_rel AS tpp
ON (tpp.project_id = p.id AND
tpp.team_id = p.team_id AND
tpp.profile_id = ?)
WHERE p.team_id = ?
AND t.deleted_at is null
ORDER BY p.modified_at DESC")
(defn get-projects
[conn profile-id team-id]
(db/exec! conn [sql:projects profile-id team-id]))
(def ^:private schema:get-projects
[:map {:title "get-projects"}
@@ -78,32 +98,11 @@
(sv/defmethod ::get-projects
{::doc/added "1.18"
::doc/changes [["2.12" "This endpoint now return deleted but recoverable projects"]]
::sm/params schema:get-projects}
[{:keys [::db/pool]} {:keys [::rpc/profile-id team-id]}]
(dm/with-open [conn (db/open pool)]
(teams/check-read-permissions! conn profile-id team-id)
(get-projects conn profile-id team-id)))
(def sql:projects
"select p.*,
coalesce(tpp.is_pinned, false) as is_pinned,
(select count(*) from file as f
where f.project_id = p.id
and deleted_at is null) as count
from project as p
inner join team as t on (t.id = p.team_id)
left join team_project_profile_rel as tpp
on (tpp.project_id = p.id and
tpp.team_id = p.team_id and
tpp.profile_id = ?)
where p.team_id = ?
and p.deleted_at is null
and t.deleted_at is null
order by p.modified_at desc")
(defn get-projects
[conn profile-id team-id]
(db/exec! conn [sql:projects profile-id team-id]))
[cfg {:keys [::rpc/profile-id team-id]}]
(teams/check-read-permissions! cfg profile-id team-id)
(get-projects cfg profile-id team-id))
;; --- QUERY: Get all projects
@@ -170,12 +169,19 @@
;; --- MUTATION: Create Project
(defn- create-project
[{:keys [::db/conn] :as cfg} {:keys [profile-id team-id] :as params}]
(let [project (teams/create-project conn params)]
[{:keys [::db/conn] :as cfg} {:keys [::rpc/request-at profile-id team-id] :as params}]
(assert (ct/inst? request-at) "expect request-at assigned")
(let [params (-> params
(assoc :created-at request-at)
(assoc :modified-at request-at))
project (teams/create-project conn params)
timestamp (::rpc/request-at params)]
(teams/create-project-role conn profile-id (:id project) :owner)
(db/insert! conn :team-project-profile-rel
{:project-id (:id project)
:profile-id profile-id
:created-at timestamp
:modified-at timestamp
:team-id team-id
:is-pinned false})
(assoc project :is-pinned false)))

2
backend/src/app/rpc/commands/verify_token.clj
View File

@@ -73,7 +73,7 @@
{:id (:id profile)}))
(-> claims
(rph/with-transform (session/create-fn cfg profile-id))
(rph/with-transform (session/create-fn cfg profile))
(rph/with-meta {::audit/name "verify-profile-email"
::audit/props (audit/profile->props profile)
::audit/profile-id (:id profile)}))))

5
backend/src/app/rpc/cond.clj
View File

@@ -39,9 +39,8 @@
(defn- encode
[s]
(-> s
bh/blake2b-256
bc/bytes->b64u
bc/bytes->str))
(bh/blake2b-256)
(bc/bytes->b64-str true)))
(defn- fmt-key
[s]

110
backend/src/app/rpc/doc.clj
View File

@@ -16,6 +16,7 @@
[app.common.schema.desc-native :as smdn]
[app.common.schema.openapi :as oapi]
[app.common.schema.registry :as sr]
[app.common.uri :as u]
[app.config :as cf]
[app.http.sse :as-alias sse]
[app.loggers.webhooks :as-alias webhooks]
@@ -25,7 +26,6 @@
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]
[pretty-spec.core :as ps]
[yetti.response :as-alias yres]))
@@ -33,8 +33,8 @@
;; DOC (human readable)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- prepare-doc-context
[methods]
(defn- context
[{:keys [methods entrypoint label openapi]}]
(letfn [(fmt-spec [mdata]
(when-let [spec (ex/ignoring (s/spec (::sv/spec mdata)))]
(with-out-str
@@ -62,8 +62,10 @@
:added (::added mdata)
:changes (some->> (::changes mdata) (partition-all 2) (map vec))
:spec (fmt-spec mdata)
:entrypoint (str (cf/get :public-uri) "/api/rpc/command/" (::sv/name mdata))
:entrypoint (-> entrypoint
(u/ensure-path-slash)
(u/join (::sv/name mdata))
(str))
:params-schema-js (fmt-schema :js mdata ::sm/params)
:result-schema-js (fmt-schema :js mdata ::sm/result)
:webhook-schema-js (fmt-schema :js mdata ::sm/webhook)
@@ -72,6 +74,9 @@
:webhook-schema-clj (fmt-schema :clj mdata ::sm/webhook)})]
{:version (:main cf/version)
:label label
:entrypoint (str entrypoint)
:openapi (str openapi)
:methods
(->> methods
(map val)
@@ -80,17 +85,19 @@
(map get-context)
(sort-by (juxt :module :name)))}))
(defn- doc-handler
[context]
(defn- handler
[& {:keys [template] :as options}]
(if (contains? cf/flags :backend-api-doc)
(let [context (delay (context options))
template (or template "app/templates/api-doc.tmpl")]
(fn [request]
(let [params (:query-params request)
pstyle (:type params "js")
context (assoc @context :param-style pstyle)]
{::yres/status 200
::yres/body (-> (io/resource "app/templates/api-doc.tmpl")
(tmpl/render context))}))
::yres/body (-> (io/resource template)
(tmpl/render context))})))
(fn [_]
{::yres/status 404})))
@@ -98,8 +105,8 @@
;; OPENAPI / SWAGGER (v3.1)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn prepare-openapi-context
[methods]
(defn- openapi-context
[{:keys [methods entrypoint description]}]
(let [definitions (atom {})
options {:registry sr/default-registry
::oapi/definitions-path "#/components/schemas/"
@@ -112,7 +119,9 @@
(fn [tsx schema]
(let [schema (sm/schema schema)
example (sm/generate schema)
example (sm/encode schema example output-transformer)]
example (sm/encode schema example output-transformer)
example (json/encode example :key-fn json/write-camel-key)]
{:default
{:description "A default response"
:content
@@ -123,7 +132,9 @@
gen-params-doc
(fn [tsx schema]
(let [example (sm/generate schema)
example (sm/encode schema example output-transformer)]
example (sm/encode schema example output-transformer)
example (json/encode example :key-fn json/write-camel-key)]
{:required true
:content
{"application/json"
@@ -158,34 +169,35 @@
(map gen-method-doc)
(sort-by (juxt :module :name))
(map (fn [doc]
[(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
[(:name doc) (:repr doc)]))
(into {})))]
{:openapi "3.0.0"
:info {:version (:main cf/version)}
:servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
;; :description "penpot backend"
}]
:servers [{:url (str entrypoint)
:description (or description "")}]
:paths paths
:components {:schemas @definitions}}))
(defn openapi-json-handler
[context]
(defn- openapi-json-handler
[& {:as options}]
(if (contains? cf/flags :backend-openapi-doc)
(let [context (delay (openapi-context options))]
(fn [_]
{::yres/status 200
::yres/headers {"content-type" "application/json; charset=utf-8"}
::yres/body (json/encode @context)})
::yres/body (json/encode @context)}))
(fn [_]
{::yres/status 404})))
(defn openapi-handler
[]
(defn- openapi-handler
[& {:keys [uri label]}]
(if (contains? cf/flags :backend-openapi-doc)
(fn [_]
(let [swagger-js (slurp (io/resource "app/assets/swagger-ui-4.18.3.js"))
swagger-cs (slurp (io/resource "app/assets/swagger-ui-4.18.3.css"))
context {:public-uri (cf/get :public-uri)
context {:uri (str uri)
:label label
:swagger-js swagger-js
:swagger-css swagger-cs}]
{::yres/status 200
@@ -196,27 +208,43 @@
{::yres/status 404})))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MODULE INIT
;; ROUTES HELPER
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmethod ig/assert-key ::routes
[_ params]
(assert (sm/valid? ::rpc/methods (::rpc/methods params)) "expected valid methods"))
(defn routes
[& {:keys [label base-uri description methods]}]
(let [entrypoint
(-> base-uri
(u/ensure-path-slash)
(u/join "methods"))
openapi
(-> base-uri
(u/ensure-path-slash)
(u/join "doc/openapi"))
template
(case label
"management" "app/templates/management-api-doc.tmpl"
"main" "app/templates/main-api-doc.tmpl")]
(defmethod ig/init-key ::routes
[_ {:keys [::rpc/methods] :as cfg}]
[(let [context (delay (prepare-doc-context methods))]
[["/_doc"
{:handler (doc-handler context)
:allowed-methods #{:get}}]
["/doc"
{:handler (doc-handler context)
:allowed-methods #{:get}}]])
(let [context (delay (prepare-openapi-context methods))]
[["/openapi"
{:handler (openapi-handler)
["" {:handler (handler :methods methods
:label label
:entrypoint entrypoint
:openapi openapi
:template template)
:allowed-methods #{:get}}]
["/openapi"
{:handler (openapi-handler
:uri (u/join openapi "openapi.json")
:label label)
:allowed-methods #{:get}}]
["/openapi.json"
{:handler (openapi-json-handler context)
:allowed-methods #{:get}}]])])
{:handler (openapi-json-handler {:entrypoint entrypoint
:description description
:methods methods})
:allowed-methods #{:get}}]]))

13
backend/src/app/rpc/helpers.clj
View File

@@ -83,3 +83,16 @@
"A convenience allias for yetti.response/stream-body"
[f]
(yres/stream-body f))
(defn get-request
"Get http request from RPC params"
[params]
(assert (contains? params ::rpc/request-at) "rpc params required")
(-> (meta params)
(get ::http/request)))
(defn get-auth-data
"Get http auth-data from RPC params"
[params]
(-> (get-request params)
(get ::http/auth-data)))

49
backend/src/app/rpc/management/exporter.clj Normal file
View File

@@ -0,0 +1,49 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.rpc.management.exporter
(:require
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.media :refer [schema:upload]]
[app.rpc :as-alias rpc]
[app.rpc.doc :as doc]
[app.storage :as sto]
[app.util.services :as sv]))
;; ---- RPC METHOD: UPLOAD-TEMPFILE
(def ^:private
schema:upload-tempfile-params
[:map {:title "upload-templfile-params"}
[:content schema:upload]])
(def ^:private
schema:upload-tempfile-result
[:map {:title "upload-templfile-result"}])
(sv/defmethod ::upload-tempfile
{::doc/added "2.12"
::sm/params schema:upload-tempfile-params
::sm/result schema:upload-tempfile-result}
[cfg {:keys [::rpc/profile-id content]}]
(let [storage (sto/resolve cfg)
hash (sto/calculate-hash (:path content))
data (-> (sto/content (:path content))
(sto/wrap-with-hash hash))
content {::sto/content data
::sto/deduplicate? true
::sto/touched-at (ct/in-future {:minutes 10})
:profile-id profile-id
:content-type (:mtype content)
:bucket "tempfile"}
object (sto/put-object! storage content)]
{:id (:id object)
:uri (-> (cf/get :public-uri)
(u/join "/assets/by-id/")
(u/join (str (:id object))))}))

183
backend/src/app/rpc/management/subscription.clj Normal file
View File

@@ -0,0 +1,183 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.rpc.management.subscription
(:require
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.schema.generators :as sg]
[app.common.time :as ct]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.rpc.commands.profile :as profile]
[app.rpc.doc :as doc]
[app.util.services :as sv]))
;; ---- RPC METHOD: AUTHENTICATE
(def ^:private
schema:authenticate-params
[:map {:title "authenticate-params"}])
(def ^:private
schema:authenticate-result
[:map {:title "authenticate-result"}
[:profile-id ::sm/uuid]])
(sv/defmethod ::auth
{::doc/added "2.12"
::sm/params schema:authenticate-params
::sm/result schema:authenticate-result}
[_ {:keys [::rpc/profile-id]}]
{:profile-id profile-id})
;; ---- RPC METHOD: GET-CUSTOMER
;; FIXME: move to app.common.time
(def ^:private schema:timestamp
(sm/type-schema
{:type ::timestamp
:pred ct/inst?
:type-properties
{:title "inst"
:description "The same as :app.common.time/inst but encodes to epoch"
:error/message "should be an instant"
:gen/gen (->> (sg/small-int)
(sg/fmap (fn [v] (ct/inst v))))
:decode/string #(some-> % ct/inst)
:encode/string #(some-> % inst-ms)
:decode/json #(some-> % ct/inst)
:encode/json #(some-> % inst-ms)}}))
(def ^:private schema:subscription
[:map {:title "Subscription"}
[:id ::sm/text]
[:customer-id ::sm/text]
[:type [:enum
"unlimited"
"professional"
"enterprise"]]
[:status [:enum
"active"
"canceled"
"incomplete"
"incomplete_expired"
"past_due"
"paused"
"trialing"
"unpaid"]]
[:billing-period [:enum
"month"
"day"
"week"
"year"]]
[:quantity :int]
[:description [:maybe ::sm/text]]
[:created-at schema:timestamp]
[:start-date [:maybe schema:timestamp]]
[:ended-at [:maybe schema:timestamp]]
[:trial-end [:maybe schema:timestamp]]
[:trial-start [:maybe schema:timestamp]]
[:cancel-at [:maybe schema:timestamp]]
[:canceled-at [:maybe schema:timestamp]]
[:current-period-end [:maybe schema:timestamp]]
[:current-period-start [:maybe schema:timestamp]]
[:cancel-at-period-end :boolean]
[:cancellation-details
[:map {:title "CancellationDetails"}
[:comment [:maybe ::sm/text]]
[:reason [:maybe ::sm/text]]
[:feedback [:maybe
[:enum
"customer_service"
"low_quality"
"missing_feature"
"other"
"switched_service"
"too_complex"
"too_expensive"
"unused"]]]]]])
(def ^:private sql:get-customer-slots
"WITH teams AS (
SELECT tpr.team_id AS id,
tpr.profile_id AS profile_id
FROM team_profile_rel AS tpr
WHERE tpr.is_owner IS true
AND tpr.profile_id = ?
), teams_with_slots AS (
SELECT tpr.team_id AS id,
count(*) AS total
FROM team_profile_rel AS tpr
WHERE tpr.team_id IN (SELECT id FROM teams)
AND tpr.can_edit IS true
GROUP BY 1
ORDER BY 2
)
SELECT max(total) AS total FROM teams_with_slots;")
(defn- get-customer-slots
[cfg profile-id]
(let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
(:total result)))
(def ^:private schema:get-customer-params
[:map])
(def ^:private schema:get-customer-result
[:map
[:id ::sm/uuid]
[:name :string]
[:num-editors ::sm/int]
[:subscription {:optional true} schema:subscription]])
(sv/defmethod ::get-customer
{::doc/added "2.12"
::sm/params schema:get-customer-params
::sm/result schema:get-customer-result}
[cfg {:keys [::rpc/profile-id]}]
(let [profile (profile/get-profile cfg profile-id)]
{:id (get profile :id)
:name (get profile :fullname)
:email (get profile :email)
:num-editors (get-customer-slots cfg profile-id)
:subscription (-> profile :props :subscription)}))
;; ---- RPC METHOD: GET-CUSTOMER
(def ^:private schema:update-customer-params
[:map
[:subscription [:maybe schema:subscription]]])
(def ^:private schema:update-customer-result
[:map])
(sv/defmethod ::update-customer
{::doc/added "2.12"
::sm/params schema:update-customer-params
::sm/result schema:update-customer-result}
[cfg {:keys [::rpc/profile-id subscription]}]
(let [{:keys [props] :as profile}
(profile/get-profile cfg profile-id ::db/for-update true)
props
(assoc props :subscription subscription)]
(l/dbg :hint "update customer"
:profile-id (str profile-id)
:subscription-type (get subscription :type)
:subscription-status (get subscription :status)
:subscription-quantity (get subscription :quantity))
(db/update! cfg :profile
{:props (db/tjson props)}
{:id profile-id}
{::db/return-keys false})
nil))

3
backend/src/app/rpc/quotes.clj
View File

@@ -102,8 +102,7 @@
::wrk/label "quotes-notification"
::wrk/params {:to (vec admins)
:subject subject
:body [{:type "text/plain"
:content content}]}}))))
:body content}}))))
(defn- generic-check!
[{:keys [::db/conn ::incr ::quote-sql ::count-sql ::default ::target] :or {incr 1} :as params}]

8
backend/src/app/setup.clj
View File

@@ -22,8 +22,7 @@
(defn- generate-random-key
[]
(-> (bn/random-bytes 64)
(bc/bytes->b64u)
(bc/bytes->str)))
(bc/bytes->b64-str true)))
(defn- get-all-props
[conn]
@@ -85,12 +84,11 @@
(l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
"all sessions on each restart, it is highly recommended setting up the "
"PENPOT_SECRET_KEY environment variable")))
(let [secret (or key (generate-random-key))]
(-> (get-all-props conn)
(assoc :secret-key secret)
(assoc :tokens-key (keys/derive secret :salt "tokens"))
(assoc :management-key (keys/derive secret :salt "management"))
(update :instance-id handle-instance-id conn (db/read-only? pool)))))))
;; FIXME
(sm/register! ::props :any)
(sm/register! ::props [:map-of :keyword ::sm/any])

10
backend/src/app/setup/clock.clj
View File

@@ -14,7 +14,9 @@
[integrant.core :as ig])
(:import
java.time.Clock
java.time.Duration))
java.time.Duration
java.time.Instant
java.time.ZoneId))
(defonce current
(atom {:clock (Clock/systemDefaultZone)
@@ -36,6 +38,12 @@
[_ _]
(remove-watch current ::common))
(defn fixed
"Get fixed clock, mainly used in tests"
[instant]
(Clock/fixed ^Instant (ct/inst instant)
^ZoneId (ZoneId/of "Z")))
(defn set-offset!
[duration]
(swap! current assoc :offset (some-> duration ct/duration)))

4
backend/src/app/setup/keys.clj
View File

@@ -8,13 +8,13 @@
"Keys derivation service."
(:refer-clojure :exclude [derive])
(:require
[app.common.spec :as us]
[buddy.core.kdf :as bk]))
(defn derive
"Derive a key from secret-key"
[secret-key & {:keys [salt size] :or {size 32}}]
(us/assert! ::us/not-empty-string secret-key)
(assert (string? secret-key) "expect string")
(assert (seq secret-key) "expect string")
(let [engine (bk/engine {:key secret-key
:salt salt
:alg :hkdf

4
backend/src/app/srepl/cli.clj
View File

@@ -61,8 +61,8 @@
:is-active is-active
:password password
:props {}}]
(->> (cmd.auth/create-profile! conn params)
(cmd.auth/create-profile-rels! conn)))))))
(->> (cmd.auth/create-profile system params)
(cmd.auth/create-profile-rels conn)))))))
(defmethod exec-command "update-profile"
[{:keys [fullname email password is-active]}]

68
backend/src/app/srepl/main.clj
View File

@@ -25,6 +25,7 @@
[app.db.sql :as-alias sql]
[app.features.fdata :as fdata]
[app.features.file-snapshots :as fsnap]
[app.http.session :as session]
[app.loggers.audit :as audit]
[app.main :as main]
[app.msgbus :as mbus]
@@ -567,48 +568,12 @@
:id file-id})))
:deleted))
(defn- restore-file*
[{:keys [::db/conn]} file-id]
(db/update! conn :file
{:deleted-at nil
:has-media-trimmed false}
{:id file-id}
{::db/return-keys false})
(db/update! conn :file-media-object
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-change
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-data
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
;; Mark thumbnails to be deleted
(db/update! conn :file-thumbnail
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
(db/update! conn :file-tagged-object-thumbnail
{:deleted-at nil}
{:file-id file-id}
{::db/return-keys false})
:restored)
(defn restore-file!
"Mark a file and all related objects as not deleted"
[file-id]
(let [file-id (h/parse-uuid file-id)]
(db/tx-run! main/system
(fn [system]
(fn [{:keys [::db/conn] :as system}]
(when-let [file (db/get* system :file
{:id file-id}
{::db/remove-deleted false
@@ -622,7 +587,9 @@
:cause "explicit call to restore-file!"}
::audit/tracked-at (ct/now)})
(restore-file* system file-id))))))
(#'files/restore-file conn file-id))
:restored))))
(defn delete-project!
"Mark a project for deletion"
@@ -655,7 +622,7 @@
(doseq [{:keys [id]} (db/query conn :file
{:project-id project-id}
{::sql/columns [:id]})]
(restore-file* cfg id))
(#'files/restore-file conn id))
:restored)
@@ -877,10 +844,33 @@
:deleted-at deleted-at
:id id})))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; SSO
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn add-sso-config
[& {:keys [base-uri client-id client-secret domain]}]
(assert (and (string? base-uri) (str/starts-with? base-uri "http")) "expected a valid base-uri")
(assert (string? client-id) "expected a valid client-id")
(assert (string? client-secret) "expected a valid client-secret")
(assert (string? domain) "expected a valid domain")
(db/insert! main/system :sso-provider
{:id (uuid/next)
:type "oidc"
:client-id client-id
:client-secret client-secret
:domain domain
:base-uri base-uri}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MISC
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn decode-session-token
[token]
(session/decode-token main/system token))
(defn instrument-var
[var]
(alter-var-root var (fn [f]

16
backend/src/app/storage.clj
View File

@@ -41,6 +41,7 @@
"file-object-thumbnail"
"file-thumbnail"
"profile"
"tempfile"
"file-data"
"file-data-fragment"
"file-change"})
@@ -163,9 +164,6 @@
backend
(:metadata result))))
(def ^:private sql:retrieve-storage-object
"select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
(defn row->storage-object [res]
(let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
(impl/storage-object
@@ -177,9 +175,15 @@
(keyword (:backend res))
mdata)))
(defn- retrieve-database-object
(def ^:private sql:get-storage-object
"SELECT *
FROM storage_object
WHERE id = ?
AND (deleted_at IS NULL)")
(defn- get-database-object
[conn id]
(some-> (db/exec-one! conn [sql:retrieve-storage-object id])
(some-> (db/exec-one! conn [sql:get-storage-object id])
(row->storage-object)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -202,7 +206,7 @@
(defn get-object
[{:keys [::db/connectable] :as storage} id]
(assert (valid-storage? storage))
(retrieve-database-object connectable id))
(get-database-object connectable id))
(defn put-object!
"Creates a new object with the provided content."

37
backend/src/app/storage/gc_deleted.clj
View File

@@ -37,7 +37,6 @@
(into #{} (map :id))
(not-empty))))
(def ^:private sql:delete-sobjects
"DELETE FROM storage_object
WHERE id = ANY(?::uuid[])")
@@ -77,47 +76,37 @@
(d/group-by (comp keyword :backend) :id #{} items))
(def ^:private sql:get-deleted-sobjects
"SELECT s.* FROM storage_object AS s
"SELECT s.*
FROM storage_object AS s
WHERE s.deleted_at IS NOT NULL
AND s.deleted_at < now() - ?::interval
AND s.deleted_at <= ?
ORDER BY s.deleted_at ASC")
(defn- get-buckets
[conn min-age]
(let [age (db/interval min-age)]
[conn]
(let [now (ct/now)]
(sequence
(comp (partition-all 25)
(mapcat group-by-backend))
(db/cursor conn [sql:get-deleted-sobjects age]))))
(db/cursor conn [sql:get-deleted-sobjects now]))))
(defn- clean-deleted!
[{:keys [::db/conn ::min-age] :as cfg}]
[{:keys [::db/conn] :as cfg}]
(reduce (fn [total [backend-id ids]]
(let [deleted (delete-in-bulk! cfg backend-id ids)]
(+ total (or deleted 0))))
0
(get-buckets conn min-age)))
(get-buckets conn)))
(defmethod ig/assert-key ::handler
[_ params]
(assert (sto/valid-storage? (::sto/storage params)) "expect valid storage")
(assert (db/pool? (::db/pool params)) "expect valid storage"))
(defmethod ig/expand-key ::handler
[k v]
{k (assoc v ::min-age (ct/duration {:hours 2}))})
(defmethod ig/init-key ::handler
[_ {:keys [::min-age] :as cfg}]
(fn [{:keys [props] :as task}]
(let [min-age (ct/duration (or (:min-age props) min-age))]
[_ cfg]
(fn [_]
(db/tx-run! cfg (fn [cfg]
(let [cfg (assoc cfg ::min-age min-age)
total (clean-deleted! cfg)]
(l/inf :hint "task finished"
:min-age (ct/format-duration min-age)
:total total)
{:deleted total}))))))
(let [total (clean-deleted! cfg)]
(l/inf :hint "task finished" :total total)
{:deleted total})))))

37
backend/src/app/storage/gc_touched.clj
View File

@@ -22,6 +22,8 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.storage :as-alias sto]
[app.storage.impl :as impl]
@@ -101,14 +103,15 @@
(def ^:private sql:mark-delete-in-bulk
"UPDATE storage_object
SET deleted_at = now(),
SET deleted_at = ?,
touched_at = NULL
WHERE id = ANY(?::uuid[])")
(defn- mark-delete-in-bulk!
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:mark-delete-in-bulk ids])))
[conn deletion-delay ids]
(let [ids (db/create-array conn "uuid" ids)
now (ct/plus (ct/now) deletion-delay)]
(db/exec-one! conn [sql:mark-delete-in-bulk now ids])))
;; NOTE: A getter that retrieves the key which will be used for group
;; ids; previously we have no value, then we introduced the
@@ -137,18 +140,20 @@
(if-let [{:keys [id] :as object} (first objects)]
(if (has-refs? conn object)
(do
(l/debug :id (str id)
(l/dbg :id (str id)
:status "freeze"
:bucket bucket)
(recur (conj to-freeze id) to-delete (rest objects)))
(do
(l/debug :id (str id)
(l/dbg :id (str id)
:status "delete"
:bucket bucket)
(recur to-freeze (conj to-delete id) (rest objects))))
(do
(let [deletion-delay (if (= bucket "tempfile")
(ct/duration {:hours 2})
(cf/get-deletion-delay))]
(some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
(some->> (seq to-delete) (mark-delete-in-bulk! conn))
(some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay))
[(count to-freeze) (count to-delete)]))))
(defn- process-bucket!
@@ -160,6 +165,7 @@
"file-thumbnail" (process-objects! conn has-file-thumbnails-refs? bucket objects)
"profile" (process-objects! conn has-profile-refs? bucket objects)
"file-data" (process-objects! conn has-file-data-refs? bucket objects)
"tempfile" (process-objects! conn (constantly false) bucket objects)
(ex/raise :type :internal
:code :unexpected-unknown-reference
:hint (dm/fmt "unknown reference '%'" bucket))))
@@ -173,27 +179,27 @@
[0 0]
(d/group-by lookup-bucket identity #{} chunk)))
(def ^:private
sql:get-touched-storage-objects
(def ^:private sql:get-touched-storage-objects
"SELECT so.*
FROM storage_object AS so
WHERE so.touched_at IS NOT NULL
AND so.touched_at <= ?
ORDER BY touched_at ASC
FOR UPDATE
SKIP LOCKED
LIMIT 10")
(defn get-chunk
[conn]
(->> (db/exec! conn [sql:get-touched-storage-objects])
[conn timestamp]
(->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
(map impl/decode-row)
(not-empty)))
(defn- process-touched!
[{:keys [::db/pool] :as cfg}]
[{:keys [::db/pool ::timestamp] :as cfg}]
(loop [freezed 0
deleted 0]
(if-let [chunk (get-chunk pool)]
(if-let [chunk (get-chunk pool timestamp)]
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
(recur (long (+ freezed nfo))
(long (+ deleted ndo))))
@@ -209,5 +215,6 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [_] (process-touched! cfg)))
(fn [_]
(process-touched! (assoc cfg ::timestamp (ct/now)))))

13
backend/src/app/storage/tmp.clj
View File

@@ -79,14 +79,17 @@
;; API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn tempfile
[& {:keys [suffix prefix min-age]
(defn tempfile*
[& {:keys [suffix prefix]
:or {prefix "penpot."
suffix ".tmp"}}]
(let [attrs (fs/make-permissions "rw-r--r--")
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))
path (Files/createFile path attrs)]
(fs/delete-on-exit! path)
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))]
(Files/createFile path attrs)))
(defn tempfile
[& {:keys [min-age] :as opts}]
(let [path (tempfile* opts)]
(sp/offer! queue [path (some-> min-age ct/duration)])
path))

66
backend/src/app/tasks/objects_gc.clj
View File

@@ -18,15 +18,15 @@
(def ^:private sql:get-profiles
"SELECT id, photo_id FROM profile
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-profiles!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id]}]
(l/trc :obj "profile" :id (str id))
@@ -41,15 +41,15 @@
(def ^:private sql:get-teams
"SELECT deleted_at, id, photo_id FROM team
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-teams!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id photo-id deleted-at]}]
(l/trc :obj "team"
:id (str id)
@@ -68,15 +68,15 @@
"SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
FROM team_font_variant
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-fonts!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at] :as font}]
(l/trc :obj "font-variant"
:id (str id)
@@ -98,15 +98,15 @@
"SELECT id, deleted_at, team_id
FROM project
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-projects!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id team-id deleted-at]}]
(l/trc :obj "project"
:id (str id)
@@ -124,15 +124,15 @@
f.project_id
FROM file AS f
WHERE f.deleted_at IS NOT NULL
AND f.deleted_at < now() + ?::interval
AND f.deleted_at <= ?
ORDER BY f.deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-files!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id deleted-at project-id] :as file}]
(l/trc :obj "file"
:id (str id)
@@ -148,15 +148,15 @@
"SELECT file_id, revn, media_id, deleted_at
FROM file_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-thumbnails!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
(l/trc :obj "file-thumbnail"
:file-id (str file-id)
@@ -175,15 +175,15 @@
"SELECT file_id, object_id, media_id, deleted_at
FROM file_tagged_object_thumbnail
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn delete-file-object-thumbnails!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
(l/trc :obj "file-object-thumbnail"
:file-id (str file-id)
@@ -203,15 +203,15 @@
"SELECT id, file_id, media_id, thumbnail_id, deleted_at
FROM file_media_object
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-media-objects!
[{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
(->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
(l/trc :obj "file-media-object"
:id (str id)
@@ -231,16 +231,15 @@
"SELECT file_id, id, type, deleted_at, metadata, backend
FROM file_data
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-data!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
(some->> metadata
@@ -266,15 +265,15 @@
"SELECT id, file_id, deleted_at
FROM file_change
WHERE deleted_at IS NOT NULL
AND deleted_at < now() + ?::interval
AND deleted_at <= ?
ORDER BY deleted_at ASC
LIMIT ?
FOR UPDATE
SKIP LOCKED")
(defn- delete-file-changes!
[{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
[{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
(->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5})
(reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
(l/trc :obj "file-change"
:id (str id)
@@ -322,9 +321,8 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [threshold (ct/duration (get props :deletion-threshold 0))
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
(fn [_]
(let [cfg (assoc cfg ::timestamp (ct/now))]
(loop [procs (map deref deletion-proc-vars)
total 0]
(if-let [proc-fn (first procs)]

12
backend/src/app/tokens.clj
View File

@@ -15,8 +15,9 @@
[buddy.sign.jwe :as jwe]))
(defn generate
[{:keys [::setup/props] :as cfg} claims]
(assert (contains? cfg ::setup/props))
([cfg claims] (generate cfg claims nil))
([{:keys [::setup/props] :as cfg} claims header]
(assert (contains? props :tokens-key) "expect props to have tokens-key")
(let [tokens-key
(get props :tokens-key)
@@ -27,7 +28,12 @@
(d/without-nils)
(t/encode))]
(jwe/encrypt payload tokens-key {:alg :a256kw :enc :a256gcm})))
(jwe/encrypt payload tokens-key {:alg :a256kw :enc :a256gcm :header header}))))
(defn decode-header
[token]
(ex/ignoring
(jwe/decode-header token)))
(defn decode
[{:keys [::setup/props] :as cfg} token]

2
backend/src/app/util/services.clj
View File

@@ -27,7 +27,7 @@
(throw (IllegalArgumentException. "Missing arguments on `defmethod` macro.")))
(let [mdata (assoc mdata
::docstring (some-> docs str/<<-)
::docstring (some-> docs str/unindent)
::spec sname
::name (name sname))

2
backend/test/backend_tests/email_sending_test.clj
View File

@@ -22,4 +22,4 @@
(t/is (contains? result :body))
(t/is (contains? result :to))
#_(t/is (contains? result :reply-to))
(t/is (vector? (:body result)))))
(t/is (map? (:body result)))))

26
backend/test/backend_tests/helpers.clj
View File

@@ -30,6 +30,7 @@
[app.rpc.commands.files :as files]
[app.rpc.commands.files-create :as files.create]
[app.rpc.commands.files-update :as files.update]
[app.rpc.commands.projects :as projects]
[app.rpc.commands.teams :as teams]
[app.rpc.helpers :as rph]
[app.util.blob :as blob]
@@ -104,13 +105,8 @@
(assoc-in [:app.rpc/methods :app.setup/templates] templates)
(dissoc :app.srepl/server
:app.http/server
:app.http/router
:app.auth.oidc.providers/google
:app.auth.oidc.providers/gitlab
:app.auth.oidc.providers/github
:app.auth.oidc.providers/generic
:app.http/route
:app.setup/templates
:app.auth.oidc/routes
:app.http.oauth/handler
:app.notifications/handler
:app.loggers.mattermost/reporter
@@ -182,23 +178,25 @@
:is-demo false}
params)]
(db/run! system
(fn [{:keys [::db/conn]}]
(fn [{:keys [::db/conn] :as cfg}]
(->> params
(cmd.auth/create-profile! conn)
(cmd.auth/create-profile-rels! conn)))))))
(cmd.auth/create-profile cfg)
(cmd.auth/create-profile-rels conn)))))))
(defn create-project*
([i params] (create-project* *system* i params))
([system i {:keys [profile-id team-id] :as params}]
(us/assert uuid? profile-id)
(us/assert uuid? team-id)
(assert (uuid? profile-id))
(assert (uuid? team-id))
(let [timestamp (ct/now)]
(db/run! system
(fn [{:keys [::db/conn]}]
(fn [cfg]
(->> (merge {:id (mk-uuid "project" i)
:name (str "project" i)}
params)
(#'teams/create-project conn))))))
params
{::rpc/request-at timestamp})
(#'projects/create-project cfg)))))))
(defn create-file*
([i params]

15
backend/test/backend_tests/http_management_test.clj
View File

@@ -22,17 +22,6 @@
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(t/deftest authenticate-method
(let [profile (th/create-profile* 1)
token (#'sess/gen-token th/*system* {:profile-id (:id profile)})
request {:params {:token token}}
response (#'mgmt/authenticate th/*system* request)]
(t/is (= 200 (::yres/status response)))
(t/is (= "authentication" (-> response ::yres/body :iss)))
(t/is (= (:id profile) (-> response ::yres/body :uid)))))
(t/deftest get-customer-method
(let [profile (th/create-profile* 1)
request {:params {:id (:id profile)}}
@@ -89,7 +78,3 @@
(let [subs' (-> response ::yres/body :subscription)]
(t/is (= subs' subs))))))

57
backend/test/backend_tests/http_middleware_access_token_test.clj
View File

@@ -1,57 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns backend-tests.http-middleware-access-token-test
(:require
[app.db :as db]
[app.http.access-token]
[app.main :as-alias main]
[app.rpc :as-alias rpc]
[app.rpc.commands.access-token]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(t/deftest soft-auth-middleware
(let [profile (th/create-profile* 1)
token (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
request (volatile! nil)
handler (#'app.http.access-token/wrap-soft-auth
(fn [req] (vreset! request req))
th/*system*)]
(with-mocks [m1 {:target 'app.http.access-token/get-token
:return nil}]
(handler {})
(t/is (= {} @request)))
(with-mocks [m1 {:target 'app.http.access-token/get-token
:return (:token token)}]
(handler {})
(let [token-id (get @request :app.http.access-token/id)]
(t/is (= token-id (:id token)))))))
(t/deftest authz-middleware
(let [profile (th/create-profile* 1)
token (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
request (volatile! {})
handler (#'app.http.access-token/wrap-authz
(fn [req] (vreset! request req))
th/*system*)]
(handler nil)
(t/is (nil? @request))
(handler {:app.http.access-token/id (:id token)})
(t/is (= #{} (:app.http.access-token/perms @request)))
(t/is (= (:id profile) (:app.http.access-token/profile-id @request)))))

135
backend/test/backend_tests/http_middleware_test.clj Normal file
View File

@@ -0,0 +1,135 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns backend-tests.http-middleware-test
(:require
[app.common.time :as ct]
[app.db :as db]
[app.http :as-alias http]
[app.http.access-token]
[app.http.middleware :as mw]
[app.http.session :as session]
[app.main :as-alias main]
[app.rpc :as-alias rpc]
[app.rpc.commands.access-token]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]
[yetti.request :as yreq]
[yetti.response :as yres]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(defrecord DummyRequest [headers cookies]
yreq/IRequestCookies
(get-cookie [_ name]
{:value (get cookies name)})
yreq/IRequest
(get-header [_ name]
(get headers name)))
(t/deftest auth-middleware-1
(let [request (volatile! nil)
handler (#'app.http.middleware/wrap-auth
(fn [req] (vreset! request req))
{})]
(handler (->DummyRequest {} {}))
(t/is (nil? (::http/auth-data @request)))
(handler (->DummyRequest {"authorization" "Token aaaa"} {}))
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
(t/is (= :token token-type))
(t/is (= "aaaa" token))
(t/is (nil? claims)))))
(t/deftest auth-middleware-2
(let [request (volatile! nil)
handler (#'app.http.middleware/wrap-auth
(fn [req] (vreset! request req))
{})]
(handler (->DummyRequest {} {}))
(t/is (nil? (::http/auth-data @request)))
(handler (->DummyRequest {"authorization" "Bearer aaaa"} {}))
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
(t/is (= :bearer token-type))
(t/is (= "aaaa" token))
(t/is (nil? claims)))))
(t/deftest auth-middleware-3
(let [request (volatile! nil)
handler (#'app.http.middleware/wrap-auth
(fn [req] (vreset! request req))
{})]
(handler (->DummyRequest {} {}))
(t/is (nil? (::http/auth-data @request)))
(handler (->DummyRequest {} {"auth-token" "foobar"}))
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
(t/is (= :cookie token-type))
(t/is (= "foobar" token))
(t/is (nil? claims)))))
(t/deftest shared-key-auth
(let [handler (#'app.http.middleware/wrap-shared-key-auth
(fn [req] {::yres/status 200})
"secret-key")]
(let [response (handler (->DummyRequest {} {}))]
(t/is (= 403 (::yres/status response))))
(let [response (handler (->DummyRequest {"x-shared-key" "secret-key2"} {}))]
(t/is (= 403 (::yres/status response))))
(let [response (handler (->DummyRequest {"x-shared-key" "secret-key"} {}))]
(t/is (= 200 (::yres/status response))))))
(t/deftest access-token-authz
(let [profile (th/create-profile* 1)
token (db/tx-run! th/*system* app.rpc.commands.access-token/create-access-token (:id profile) "test" nil)
handler (#'app.http.access-token/wrap-authz identity th/*system*)]
(let [response (handler nil)]
(t/is (nil? response)))
(let [response (handler {::http/auth-data {:type :token :token "foobar" :claims {:tid (:id token)}}})]
(t/is (= #{} (:app.http.access-token/perms response)))
(t/is (= (:id profile) (:app.http.access-token/profile-id response))))))
(t/deftest session-authz
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
handler (-> (fn [req] req)
(#'session/wrap-authz {::session/manager manager})
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
response (handler (->DummyRequest {} {"auth-token" (:token session)}))
{:keys [token claims] token-type :type}
(get response ::http/auth-data)]
(t/is (= :cookie token-type))
(t/is (= (:token session) token))
(t/is (= "authentication" (:iss claims)))
(t/is (= "penpot" (:aud claims)))
(t/is (= (:id session) (:sid claims)))
(t/is (= (:id profile) (:uid claims)))))

2
backend/test/backend_tests/rpc_doc_test.clj
View File

@@ -23,7 +23,7 @@
(smt/check!
(smt/for [context (->> sg/int
(sg/fmap (fn [_]
(rpc.doc/prepare-openapi-context (::rpc/methods th/*system*)))))]
(#'rpc.doc/openapi-context (::rpc/methods th/*system*)))))]
(try
(json/encode context)
true

7
backend/test/backend_tests/rpc_file_snapshot_test.clj
View File

@@ -9,6 +9,7 @@
[app.common.features :as cfeat]
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -16,6 +17,7 @@
[app.db.sql :as sql]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -132,9 +134,10 @@
;; this will run pending task triggered by deleting user snapshot
(th/run-pending-tasks!)
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
;; delete 2 snapshots and 2 file data entries
(t/is (= 4 (:processed res))))))))
(t/is (= 4 (:processed res)))))))))
(t/deftest snapshots-locking
(let [profile-1 (th/create-profile* 1 {:is-active true})

273
backend/test/backend_tests/rpc_file_test.clj
View File

@@ -19,6 +19,7 @@
[app.http :as http]
[app.rpc :as-alias rpc]
[app.rpc.commands.files :as files]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -142,20 +143,6 @@
(t/is (= 0 (count result))))))))
(t/deftest file-gc-with-fragments
(letfn [(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
(let [params {::th/type :update-file
::rpc/profile-id profile-id
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(:result out)))]
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)
@@ -261,7 +248,7 @@
(t/is (= 2 (:processed res))))
(let [rows (th/db-query :file-data {:file-id (:id file) :type "fragment"})]
(t/is (= 2 (count rows)))))))
(t/is (= 2 (count rows))))))
(t/deftest file-gc-with-thumbnails
(letfn [(add-file-media-object [& {:keys [profile-id file-id]}]
@@ -279,20 +266,6 @@
;; (th/print-result! out)
(t/is (nil? (:error out)))
(:result out)))
(update-file! [& {:keys [profile-id file-id changes revn] :or {revn 0}}]
(let [params {::th/type :update-file
::rpc/profile-id profile-id
:id file-id
:session-id (uuid/random)
:revn revn
:vern 0
:features cfeat/supported-features
:changes changes}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(:result out)))]
(let [storage (:app.storage/storage th/*system*)
@@ -340,7 +313,7 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; the same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -399,14 +372,14 @@
(th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
(th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file change and file data entries for two snapshots
(t/is (= 4 (:processed res))))
;; Rerun the file-gc and objects-gc
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
(let [res (th/run-task! :objects-gc {})]
;; this will remove the file media objects marked as deleted
;; on prev file-gc
(t/is (= 2 (:processed res))))
@@ -414,7 +387,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -599,7 +572,7 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -692,7 +665,7 @@
;; because of the deduplication (we have uploaded 2 times the
;; same files).
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -742,7 +715,7 @@
;; Now that objects-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! "storage-gc-touched" {})]
(t/is (= 1 (:freeze res))))
;; check file media objects
@@ -777,7 +750,7 @@
;; Now that file-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete res))))
;; check file media objects
@@ -949,8 +922,9 @@
(t/is (= 0 (:processed result))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 3 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result)))))
;; query the list of file libraries of a after hard deletion
(let [data {::th/type :get-file-libraries
@@ -1161,7 +1135,7 @@
(th/sleep 300)
;; run the task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that object thumbnails are still here
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1190,7 +1164,7 @@
(t/is (= 2 (count rows))))
;; run the task again
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; check that we have all object thumbnails
(let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1253,7 +1227,7 @@
(t/is (= 2 (count rows)))))
(t/testing "gc task"
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
(t/is (= 2 (count rows)))
@@ -1300,7 +1274,7 @@
;; The FileGC task will schedule an inner taskq
(th/run-pending-tasks!)
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -1394,7 +1368,7 @@
;; we ensure that once object-gc is passed and marked two storage
;; objects to delete
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@@ -1516,7 +1490,7 @@
(t/is (some? (not-empty (:objects component))))))
;; Re-run the file-gc task
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [row (th/db-get :file {:id (:id file)})]
(t/is (true? (:has-media-trimmed row))))
@@ -1546,7 +1520,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1637,8 +1611,8 @@
:component-id c-id})}])
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Check that component exists
(let [data {::th/type :get-file
@@ -1711,7 +1685,7 @@
;; Now, we have deleted the usage of component if we pass file-gc,
;; that component should be deleted
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
;; Check that component is properly removed
(let [data {::th/type :get-file
@@ -1860,8 +1834,8 @@
(t/is (not= (:id fill) (:id fmedia)))))
;; Run the file-gc on file and library
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
;; Now proceed to delete file and absorb it
(let [data {::th/type :delete-file
@@ -1893,3 +1867,200 @@
(t/is (= (:id file-2) (:file-id (get rows 0))))
(t/is (nil? (:deleted-at (get rows 0)))))))
(t/deftest deleted-files-permanently-delete
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
file-id (uuid/next)
now (ct/inst "2025-10-31T00:00:00Z")]
(binding [ct/*clock* (clock/fixed now)]
(let [data {::th/type :create-file
::rpc/profile-id (:id prof)
:project-id proj-id
:id file-id
:name "foobar"
:is-shared false
:components-v2 true}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= (:name data) (:name result)))
(t/is (= proj-id (:project-id result)))))
(let [data {::th/type :delete-file
:id file-id
::rpc/profile-id (:id prof)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
;; get deleted files
(let [data {::th/type :get-team-deleted-files
::rpc/profile-id (:id prof)
:team-id team-id}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [[row1 :as result] (:result out)]
(t/is (= 1 (count result)))
(t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
(let [data {::th/type :permanently-delete-team-files
::rpc/profile-id (:id prof)
:team-id team-id
:ids #{file-id}}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= (:ids data) result)))
(let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
(t/is (= (:deleted-at row) now)))))))
(t/deftest restore-deleted-files
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
file-id (uuid/next)
now (ct/inst "2025-10-31T00:00:00Z")]
(binding [ct/*clock* (clock/fixed now)]
(let [data {::th/type :create-file
::rpc/profile-id (:id prof)
:project-id proj-id
:id file-id
:name "foobar"
:is-shared false
:components-v2 true}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= (:name data) (:name result)))
(t/is (= proj-id (:project-id result)))))
(let [data {::th/type :delete-file
:id file-id
::rpc/profile-id (:id prof)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
;; get deleted files
(let [data {::th/type :get-team-deleted-files
::rpc/profile-id (:id prof)
:team-id team-id}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [[row1 :as result] (:result out)]
(t/is (= 1 (count result)))
(t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
(let [data {::th/type :restore-deleted-team-files
::rpc/profile-id (:id prof)
:team-id team-id
:ids #{file-id}}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (fn? result))
(let [events (th/consume-sse result)]
;; (pp/pprint events)
(t/is (= 2 (count events)))
(t/is (= :end (first (last events))))
(t/is (= (:ids data) (last (last events)))))))
(let [row (th/db-exec-one! ["select * from file where id = ?" file-id])]
(t/is (nil? (:deleted-at row)))))))
(t/deftest restore-deleted-files-and-projets
(let [profile (th/create-profile* 1 {:is-active true})
team-id (:default-team-id profile)
now (ct/inst "2025-10-31T00:00:00Z")]
(binding [ct/*clock* (clock/fixed now)]
(let [project (th/create-project* 1 {:profile-id (:id profile)
:team-id team-id})
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:id project)})
data {::th/type :delete-project
:id (:id project)
::rpc/profile-id (:id profile)}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out)))
(th/run-pending-tasks!)
;; get deleted files
(let [data {::th/type :get-team-deleted-files
::rpc/profile-id (:id profile)
:team-id team-id}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [[row1 :as result] (:result out)]
(t/is (= 1 (count result)))
(t/is (= (:will-be-deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z"))))
;; Check if project is deleted
(let [[row1 :as rows] (th/db-query :project {:id (:id project)})]
;; (pp/pprint rows)
(t/is (= 1 (count rows)))
(t/is (= (:deleted-at row1) #penpot/inst "2025-11-07T00:00:00Z"))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (= (:modified-at row1) #penpot/inst "2025-10-31T00:00:00Z")))
;; Restore files
(let [data {::th/type :restore-deleted-team-files
::rpc/profile-id (:id profile)
:team-id team-id
:ids #{(:id file)}}
out (th/command! data)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (fn? result))
(let [events (th/consume-sse result)]
;; (pp/pprint events)
(t/is (= 2 (count events)))
(t/is (= :end (first (last events))))
(t/is (= (:ids data) (last (last events)))))))
(let [[row1 :as rows] (th/db-query :file {:project-id (:id project)})]
;; (pp/pprint rows)
(t/is (= 1 (count rows)))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (nil? (:deleted-at row1))))
;; Check if project is restored
(let [[row1 :as rows] (th/db-query :project {:id (:id project)})]
;; (pp/pprint rows)
(t/is (= 1 (count rows)))
(t/is (= (:created-at row1) #penpot/inst "2025-10-31T00:00:00Z"))
(t/is (nil? (:deleted-at row1))))))))

28
backend/test/backend_tests/rpc_file_thumbnails_test.clj
View File

@@ -8,12 +8,14 @@
(:require
[app.common.pprint :as pp]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape :as cts]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.rpc.commands.auth :as cauth]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -83,7 +85,8 @@
(t/is (map? (:result out))))
;; run the task again
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! "storage-gc-touched" {}))]
(t/is (= 2 (:freeze res))))
(let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
@@ -114,9 +117,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(th/run-task! :file-gc {:min-age 0 :file-id (:id file)})
(th/run-task! :file-gc {:file-id (:id file)})
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 3 (:processed result))))
;; check if row2 related thumbnail row still exists
@@ -133,7 +136,8 @@
(t/is (some? (sto/get-object storage (:media-id row2))))
;; run the task again
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete res)))
(t/is (= 0 (:freeze res))))
@@ -143,8 +147,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(t/is (nil? (sto/get-object storage (:media-id row1))))
(t/is (some? (sto/get-object storage (:media-id row2))))
@@ -216,9 +221,9 @@
;; Run the File GC task that should remove unused file object
;; thumbnails
(t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
(t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
(let [result (th/run-task! :objects-gc {:min-age 0})]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result))))
;; check if row1 related thumbnail row still exists
@@ -230,7 +235,7 @@
(t/is (= (:object-id data1) (:object-id row)))
(t/is (uuid? (:media-id row1))))
(let [result (th/run-task! :storage-gc-touched {:min-age 0})]
(let [result (th/run-task! :storage-gc-touched {})]
(t/is (= 1 (:delete result))))
;; Check if storage objects still exists after file-gc
@@ -242,8 +247,9 @@
;; Run the storage gc deleted task, it should permanently delete
;; all storage objects related to the deleted thumbnails
(let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
(t/is (= 1 (:deleted result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted result)))))
(t/is (some? (sto/get-object storage (:media-id row2)))))))

35
backend/test/backend_tests/rpc_font_test.clj
View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-font-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -129,7 +131,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -141,16 +143,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res))))))
(t/is (= 6 (:delete res)))))))
(t/deftest font-deletion-2
(let [prof (th/create-profile* 1 {:is-active true})
@@ -189,7 +192,7 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@@ -201,16 +204,17 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(t/is (= 3 (:delete res)))))))
(t/deftest font-deletion-3
(let [prof (th/create-profile* 1 {:is-active true})
@@ -248,7 +252,7 @@
(t/is (nil? (:error out1)))
(t/is (nil? (:error out2)))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font-variant
@@ -260,13 +264,14 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(let [res (th/run-task! :storage-gc-touched {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res))))))
(t/is (= 3 (:delete res)))))))

13
backend/test/backend_tests/rpc_project_test.clj
View File

@@ -6,11 +6,13 @@
(ns backend-tests.rpc-project-test
(:require
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[backend-tests.helpers :as th]
[clojure.test :as t]))
@@ -104,7 +106,8 @@
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= 1 (count result)))))))
(t/is (= 1 (count (remove :deleted-at result))))
(t/is (= 2 (count result)))))))
(t/deftest permissions-checks-create-project
(let [profile1 (th/create-profile* 1)
@@ -207,7 +210,8 @@
;; (th/print-result! out)
(t/is (nil? (:error out)))
(let [result (:result out)]
(t/is (= 1 (count result)))))
(t/is (= 2 (count result)))
(t/is (= 1 (count (remove :deleted-at result))))))
;; run permanent deletion (should be noop)
(let [result (th/run-task! :objects-gc {})]
@@ -224,8 +228,9 @@
(t/is (= 0 (count result)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 1 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed result)))))
;; query the list of files of a after hard deletion
(let [data {::th/type :get-project-files

11
backend/test/backend_tests/rpc_team_test.clj
View File

@@ -13,6 +13,7 @@
[app.db :as db]
[app.http :as http]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[app.tokens :as tokens]
[backend-tests.helpers :as th]
@@ -525,8 +526,9 @@
(t/is (= :not-found (:type edata)))))
;; run permanent deletion
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 2 (:processed result))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed result)))))
;; query the list of projects of a after hard deletion
(let [data {::th/type :get-projects
@@ -581,8 +583,9 @@
(t/is (= 1 (count rows)))
(t/is (ct/inst? (:deleted-at (first rows)))))
(let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
(t/is (= 7 (:processed result))))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
(let [result (th/run-task! :objects-gc {})]
(t/is (= 7 (:processed result)))))))
(t/deftest create-team-access-request
(with-mocks [mock {:target 'app.email/send! :return nil}]

80
backend/test/backend_tests/storage_test.clj
View File

@@ -11,6 +11,7 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.rpc :as-alias rpc]
[app.setup.clock :as clock]
[app.storage :as sto]
[backend-tests.helpers :as th]
[clojure.test :as t]
@@ -53,19 +54,13 @@
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
::sto/expired-at (ct/in-future {:seconds 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(t/is (sto/object? object))
(t/is (ct/inst? (:expired-at object)))
(t/is (ct/is-after? (:expired-at object) (ct/now)))
(t/is (= object (sto/get-object storage (:id object))))
(th/sleep 1000)
(t/is (nil? (sto/get-object storage (:id object))))
(t/is (nil? (sto/get-object-data storage object)))
(t/is (nil? (sto/get-object-url storage object)))
(t/is (nil? (sto/get-object-path storage object)))))
(t/is (nil? (sto/get-object storage (:id object))))))
(t/deftest put-and-delete-object
(let [storage (-> (:app.storage/storage th/*system*)
@@ -98,20 +93,25 @@
::sto/expired-at (ct/now)
:content-type "text/plain"})
object2 (sto/put-object! storage {::sto/content content2
::sto/expired-at (ct/in-past {:hours 2})
::sto/expired-at (ct/in-future {:hours 2})
:content-type "text/plain"})
object3 (sto/put-object! storage {::sto/content content3
::sto/expired-at (ct/in-past {:hours 1})
::sto/expired-at (ct/in-future {:hours 1})
:content-type "text/plain"})]
(th/sleep 200)
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res))))
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 2 (:count res))))))
(t/is (= 2 (:count res))))
(binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 1 (:deleted res)))))
(let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
(t/is (= 1 (:count res))))))
(t/deftest touched-gc-task-1
(let [storage (-> (:app.storage/storage th/*system*)
@@ -158,7 +158,7 @@
{:id (:id result-1)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; check that we still have all the storage objects
@@ -182,7 +182,6 @@
(let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
(t/is (= 0 (:count res)))))))
(t/deftest touched-gc-task-2
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@@ -243,11 +242,12 @@
{:id (:id result-2)})
;; run the objects gc task for permanent deletion
(let [res (th/run-task! :objects-gc {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
;; revert touched state to all storage objects
(th/db-exec-one! ["update storage_object set touched_at=now()"])
(th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
;; Run the task again
(let [res (th/run-task! :storage-gc-touched {})]
@@ -293,10 +293,10 @@
result-2 (:result out2)]
;; now we proceed to manually mark all storage objects touched
(th/db-exec! ["update storage_object set touched_at=now()"])
(th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@@ -305,16 +305,48 @@
(t/is (= 2 (count rows)))))
;; now we proceed to manually delete all file_media_object
(th/db-exec! ["update file_media_object set deleted_at = now()"])
(th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)])
(let [res (th/run-task! "objects-gc" {:min-age 0})]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
;; check that we have all no objects
(let [rows (th/db-exec! ["select * from storage_object where deleted_at is null"])]
(t/is (= 0 (count rows))))))
(t/deftest tempfile-bucket-test
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content1 (sto/content "content1")
now (ct/now)
object1 (sto/put-object! storage {::sto/content content1
::sto/touched-at (ct/plus now {:minutes 1})
:bucket "tempfile"
:content-type "text/plain"})]
(binding [ct/*clock* (clock/fixed now)]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res)))))
(binding [ct/*clock* (clock/fixed (ct/plus now {:minutes 1}))]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 1 (:delete res)))))
(binding [ct/*clock* (clock/fixed (ct/plus now {:hours 1}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 0 (:deleted res)))))
(binding [ct/*clock* (clock/fixed (ct/plus now {:hours 2}))]
(let [res (th/run-task! :storage-gc-deleted {})]
(t/is (= 0 (:deleted res)))))))

6
common/deps.edn
View File

@@ -17,7 +17,7 @@
org.slf4j/slf4j-api {:mvn/version "2.0.17"}
pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.40"}
selmer/selmer {:mvn/version "1.12.62"}
selmer/selmer {:mvn/version "1.12.69"}
criterium/criterium {:mvn/version "0.4.6"}
metosin/jsonista {:mvn/version "0.3.13"}
@@ -48,12 +48,8 @@
com.sun.mail/jakarta.mail {:mvn/version "2.0.2"}
org.la4j/la4j {:mvn/version "0.6.0"}
;; exception printing
fipp/fipp {:mvn/version "0.6.29"}
me.flowthing/pp {:mvn/version "2024-11-13.77"}
io.aviso/pretty {:mvn/version "1.4.4"}
environ/environ {:mvn/version "1.2.0"}}
:paths ["src" "vendor" "target/classes"]

7
common/scripts/test Executable file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -ex
corepack enable;
corepack install;
yarn install;
yarn run test;

6
common/src/app/common/data.cljc
View File

@@ -1048,6 +1048,12 @@
(into [elem])
(into (subvec without-elem insert-pos)))))))
(defn invert-map
"Returns a map with keys and values swapped.
If the input map has duplicate values, later entries overwrite earlier ones."
[m]
(into {} (map (fn [[k v]] [v k]) m)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; String Functions
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

7
common/src/app/common/data/macros.cljc
View File

@@ -9,10 +9,10 @@
(:refer-clojure :exclude [get-in select-keys str with-open max])
#?(:cljs (:require-macros [app.common.data.macros]))
(:require
#?(:clj [cljs.analyzer.api :as aapi])
#?(:clj [clojure.core :as c]
:cljs [cljs.core :as c])
[app.common.data :as d]
[cljs.analyzer.api :as aapi]
[cuerdas.core :as str]))
(defmacro select-keys
@@ -44,7 +44,8 @@
[& params]
`(str/concat ~@params))
(defmacro export
#?(:clj
(defmacro export
"A helper macro that allows reexport a var in a current namespace."
[v]
(if (boolean (:ns &env))
@@ -79,7 +80,7 @@
(alter-meta! (var ~n) merge (dissoc m# :name))
;; (when (:macro m#)
;; (.setMacro (var ~n)))
~vr))))
~vr)))))
(defmacro fmt
"String interpolation helper. Can only be used with strings known at

3
common/src/app/common/features.cljc
View File

@@ -53,6 +53,7 @@
"plugins/runtime"
"tokens/numeric-input"
"design-tokens/v1"
"text-editor/v2-html-paste"
"text-editor/v2"
"render-wasm/v1"
"variants/v1"})
@@ -75,6 +76,7 @@
(def frontend-only-features
#{"styles/v2"
"plugins/runtime"
"text-editor/v2-html-paste"
"text-editor/v2"
"tokens/numeric-input"
"render-wasm/v1"})
@@ -124,6 +126,7 @@
:feature-plugins "plugins/runtime"
:feature-design-tokens "design-tokens/v1"
:feature-text-editor-v2 "text-editor/v2"
:feature-text-editor-v2-html-paste "text-editor/v2-html-paste"
:feature-render-wasm "render-wasm/v1"
:feature-variants "variants/v1"
:feature-token-input "tokens/numeric-input"

7
common/src/app/common/files/builder.cljc
View File

@@ -485,6 +485,13 @@
(commit-change change1)
(commit-change change2))))
(defn add-tokens-lib
[state tokens-lib]
(-> state
(commit-change
{:type :set-tokens-lib
:tokens-lib tokens-lib})))
(defn delete-shape
[file id]
(commit-change

177
common/src/app/common/files/changes.cljc
View File

@@ -371,7 +371,7 @@
[:set-tokens-lib
[:map {:title "SetTokensLib"}
[:type [:= :set-tokens-lib]]
[:tokens-lib ::sm/any]]] ;; TODO: we should define a plain object schema for tokens-lib
[:tokens-lib [:maybe ctob/schema:tokens-lib]]]]
[:set-token
[:map {:title "SetTokenChange"}
@@ -463,35 +463,16 @@
;; Changes Processing Impl
(defn validate-shapes!
[data-old data-new items]
(letfn [(validate-shape! [[page-id id]]
(let [shape-old (dm/get-in data-old [:pages-index page-id :objects id])
shape-new (dm/get-in data-new [:pages-index page-id :objects id])]
;; If object has changed or is new verify is correct
(when (and (some? shape-new)
(not= shape-old shape-new))
(when-not (and (cts/valid-shape? shape-new)
(cts/shape? shape-new))
#_:clj-kondo/ignore
(defn- validate-shape
[{:keys [id] :as shape} page-id]
(when-not (cts/valid-shape? shape)
(ex/raise :type :assertion
:code :data-validation
:hint (str "invalid shape found after applying changes on file "
(:id data-new))
:file-id (:id data-new)
::sm/explain (cts/explain-shape shape-new))))))]
(->> (into #{} (map :page-id) items)
(mapcat (fn [page-id]
(filter #(= page-id (:page-id %)) items)))
(mapcat (fn [{:keys [type id page-id] :as item}]
(sequence
(map (partial vector page-id))
(case type
(:add-obj :mod-obj :del-obj) (cons id nil)
(:mov-objects :reg-objects) (:shapes item)
nil))))
(run! validate-shape!))))
:hint (str "invalid shape found '" id "'")
:page-id page-id
:shape-id id
::sm/explain (cts/explain-shape shape))))
(defn- process-touched-change
[data {:keys [id page-id component-id]}]
@@ -517,16 +498,9 @@
(when verify?
(check-changes items))
(binding [*touched-changes* (volatile! #{})
cts/*wasm-sync* true]
(let [result (reduce #(or (process-change %1 %2) %1) data items)
result (reduce process-touched-change result @*touched-changes*)]
;; Validate result shapes (only on the backend)
;;
;; TODO: (PERF) add changed shapes tracking and only validate
;; the tracked changes instead of iterate over all shapes
#?(:clj (validate-shapes! data result items))
result))))
(binding [*touched-changes* (volatile! #{})]
(let [result (reduce #(or (process-change %1 %2) %1) data items)]
(reduce process-touched-change result @*touched-changes*)))))
;; --- Comment Threads
@@ -614,9 +588,10 @@
(defmethod process-change :add-obj
[data {:keys [id obj page-id component-id frame-id parent-id index ignore-touched]}]
(let [update-container
(fn [container]
(ctst/add-shape id obj container frame-id parent-id index ignore-touched))]
;; NOTE: we only perform hard validation on backend
#?(:clj (validate-shape obj page-id))
(let [update-container #(ctst/add-shape id obj % frame-id parent-id index ignore-touched)]
(when *state*
(swap! *state* collect-shape-media-refs obj page-id))
@@ -639,6 +614,9 @@
(when (and *state* page-id)
(swap! *state* collect-shape-media-refs shape page-id))
;; NOTE: we only perform hard validation on backend
#?(:clj (validate-shape shape page-id))
(assoc objects id shape))
objects))
@@ -693,8 +671,6 @@
(d/update-in-when data [:pages-index page-id] fix-container)
(d/update-in-when data [:components component-id] fix-container))))
;; FIXME: remove, seems like this method is already unused
;; reg-objects operation "regenerates" the geometry and selrect of the parent groups
(defmethod process-change :reg-objects
[data {:keys [page-id component-id shapes]}]
;; FIXME: Improve performance
@@ -723,8 +699,8 @@
(update-group [group objects]
(let [lookup (d/getf objects)
children (get group :shapes)]
(cond
children (get group :shapes)
group (cond
;; If the group is empty we don't make any changes. Will be removed by a later process
(empty? children)
group
@@ -738,33 +714,45 @@
:else
(->> (map lookup children)
(gsh/update-group-selrect group)))))]
(gsh/update-group-selrect group)))]
#?(:clj (validate-shape group page-id))
group))]
(if page-id
(d/update-in-when data [:pages-index page-id :objects] reg-objects)
(d/update-in-when data [:components component-id :objects] reg-objects))))
(defmethod process-change :mov-objects
[data {:keys [parent-id shapes index page-id component-id ignore-touched after-shape allow-altering-copies syncing]}]
;; FIXME: ignore-touched is no longer used, so we can consider it deprecated
[data {:keys [parent-id shapes index page-id component-id #_ignore-touched after-shape allow-altering-copies syncing]}]
(letfn [(calculate-invalid-targets [objects shape-id]
(let [reduce-fn #(into %1 (calculate-invalid-targets objects %2))]
(->> (get-in objects [shape-id :shapes])
(reduce reduce-fn #{shape-id}))))
;; Avoid placing a shape as a direct or indirect child of itself,
;; or inside its main component if it's in a copy,
;; or inside a copy, or from a copy
;; Avoid placing a shape as a direct or indirect child of itself, or
;; inside its main component if it's in a copy, or inside a copy, or
;; from a copy
(is-valid-move? [objects shape-id]
(let [invalid-targets (calculate-invalid-targets objects shape-id)
shape (get objects shape-id)]
(and shape
(not (invalid-targets parent-id))
(not (cfh/components-nesting-loop? objects shape-id parent-id))
(or allow-altering-copies ;; In some cases (like a component swap) it's allowed to change the structure of a copy
syncing ;; If we are syncing the changes of a main component, it's allowed to change the structure of a copy
(or
;; In some cases (like a component
;; swap) it's allowed to change the
;; structure of a copy
allow-altering-copies
;; DEPRECATED, remove once v2.12 released
syncing
(and
(not (ctk/in-component-copy? (get objects (:parent-id shape)))) ;; We don't want to change the structure of component copies
(not (ctk/in-component-copy? (get objects parent-id)))))))) ;; We need to check the origin and target frames
;; We don't want to change the structure of component copies
(not (ctk/in-component-copy? (get objects (:parent-id shape))))
;; We need to check the origin and target frames
(not (ctk/in-component-copy? (get objects parent-id))))))))
(insert-items [prev-shapes index shapes]
(let [prev-shapes (or prev-shapes [])]
@@ -773,17 +761,13 @@
(cfh/append-at-the-end prev-shapes shapes))))
(add-to-parent [parent index shapes]
(let [parent (-> parent
(update :shapes insert-items index shapes)
;; We need to ensure that no `nil` in the
;; shapes list after adding all the
;; incoming shapes to the parent.
(update :shapes d/vec-without-nils))]
(cond-> parent
(and (:shape-ref parent)
(#{:group :frame} (:type parent))
(not ignore-touched))
(dissoc :remote-synced))))
(update parent :shapes
(fn [parent-shapes]
(-> parent-shapes
(insert-items index shapes)
;; We need to ensure that no `nil` in the shapes list
;; after adding all the incoming shapes to the parent.
(d/vec-without-nils)))))
(remove-from-old-parent [old-objects objects shape-id]
(let [prev-parent-id (dm/get-in old-objects [shape-id :parent-id])]
@@ -791,43 +775,44 @@
;; the new destination target parent id.
(if (= prev-parent-id parent-id)
objects
(let [sid shape-id
pid prev-parent-id
obj (get objects pid)
component? (and (:shape-ref obj)
(= (:type obj) :group)
(not ignore-touched))]
(-> objects
(d/update-in-when [pid :shapes] d/without-obj sid)
(d/update-in-when [pid :shapes] d/vec-without-nils)
(cond-> component? (d/update-when pid #(dissoc % :remote-synced))))))))
(d/update-in-when objects [prev-parent-id :shapes]
(fn [shapes]
(-> shapes
(d/without-obj shape-id)
(d/vec-without-nils)))))))
(update-parent-id [objects id]
(-> objects
(d/update-when id assoc :parent-id parent-id)))
(d/update-when objects id assoc :parent-id parent-id))
;; Updates the frame-id references that might be outdated
(assign-frame-id [frame-id objects id]
(let [objects (d/update-when objects id assoc :frame-id frame-id)
obj (get objects id)]
(update-frame-id [frame-id objects id]
(let [obj (some-> (get objects id)
(assoc :frame-id frame-id))]
(cond-> objects
;; If we moving frame, the parent frame is the root
;; and we DO NOT NEED update children because the
;; children will point correctly to the frame what we
;; are currently moving
(not= :frame (:type obj))
(as-> $$ (reduce (partial assign-frame-id frame-id) $$ (:shapes obj))))))
(some? obj)
(assoc id obj)
;; If we moving a frame, we DO NOT NEED update
;; children because the children will point correctly
;; to the frame what we are currently moving
(not (cfh/frame-shape? obj))
(as-> $$ (reduce (partial update-frame-id frame-id) $$ (:shapes obj))))))
(validate-shape [objects #_:clj-kondo/ignore shape-id]
#?(:clj (when-let [shape (get objects shape-id)]
(validate-shape shape page-id)))
objects)
(move-objects [objects]
(let [valid? (every? (partial is-valid-move? objects) shapes)
parent (get objects parent-id)
after-shape-index (d/index-of (:shapes parent) after-shape)
index (if (nil? after-shape-index) index (inc after-shape-index))
frame-id (if (= :frame (:type parent))
(let [parent (get objects parent-id)]
;; Do not proceed with the move if parent does not
;; exists; this can happen on a race condition when an
;; inflight move operations lands when parent is deleted
(if (and (seq shapes) (every? (partial is-valid-move? objects) shapes) parent)
(let [index (or (some-> (d/index-of (:shapes parent) after-shape) inc) index)
frame-id (if (cfh/frame-shape? parent)
(:id parent)
(:frame-id parent))]
(if (and valid? (seq shapes))
(as-> objects $
;; Add the new shapes to the parent object.
(d/update-when $ parent-id #(add-to-parent % index shapes))
@@ -842,7 +827,11 @@
;; Ensure that all shapes of the new parent has a
;; correct link to the topside frame.
(reduce (partial assign-frame-id frame-id) $ shapes))
(reduce (partial update-frame-id frame-id) $ shapes)
;; Perform validation of the affected shapes
(reduce validate-shape $ shapes)))
objects)))]
(if page-id

1
common/src/app/common/files/changes_builder.cljc
View File

@@ -638,6 +638,7 @@
(reduce add-undo-change-shape $ ids)))
(apply-changes-local)))))
;; FIXME: PERFORMANCE
(defn resize-parents
[changes ids]
(assert-page-id! changes)

4
common/src/app/common/files/helpers.cljc
View File

@@ -72,9 +72,11 @@
(= :bool (dm/get-prop shape :type))))
(defn text-shape?
[shape]
([shape]
(and (some? shape)
(= :text (dm/get-prop shape :type))))
([objects id]
(text-shape? (get objects id))))
(defn rect-shape?
[shape]

147
common/src/app/common/files/migrations.cljc
View File

@@ -1381,17 +1381,27 @@
(defmethod migrate-data "0006-fix-old-texts-fills"
[data _]
(letfn [(fix-fills [node]
(let [fills (if (and (not (seq (:fills node)))
(let [;; In the old format refs were strings
sanitize-uuid
(fn [o]
(if (uuid? o)
o
(uuid/parse* o)))
fills
(if (and (not (seq (:fills node)))
(or (some? (:fill-color node))
(some? (:fill-opacity node))
(some? (:fill-color-gradient node))))
[(d/without-nils (select-keys node [:fill-color :fill-opacity :fill-color-gradient
:fill-color-ref-id :fill-color-ref-file]))]
[(-> (select-keys node types.fills/fill-attrs)
(update :fill-color-ref-file sanitize-uuid)
(update :fill-color-ref-id sanitize-uuid)
(d/without-nils))]
(:fills node))]
(-> node
(assoc :fills fills)
(dissoc :fill-color :fill-opacity :fill-color-gradient
:fill-color-ref-id :fill-color-ref-file))))
(reduce dissoc
(assoc node :fills fills)
types.fills/fill-attrs)))
(update-object [object]
(if (cfh/text-shape? object)
@@ -1598,6 +1608,63 @@
;; as value; this migration fixes it.
(d/update-when data :components d/update-vals d/without-nils))
(defmethod migrate-data "0015-fix-text-attrs-blank-strings"
[data _]
;; After making text validation more restrictive (using ::sm/text
;; instead of :string), we need to fix text attributes that contain
;; empty or blank strings. These should be replaced with default
;; values from default-text-attrs.
(letfn [(blank-or-empty? [v]
(or (nil? v)
(and (string? v)
(or (str/empty? v)
(str/blank? v)))))
(get-default-value [attr]
(let [defaults types.text/default-text-attrs]
(case attr
;; direction in content maps to text-direction in defaults
:direction (:text-direction defaults)
;; For other attrs, get directly from defaults
(get defaults attr))))
(fix-text-attrs [node]
;; These are the attributes that were changed to ::sm/text in the schema
(let [text-attrs [:font-family :font-size :font-style :font-weight
:direction :text-decoration :text-transform]]
(reduce
(fn [node attr]
(if (and (contains? node attr)
(blank-or-empty? (get node attr)))
;; Replace blank/empty value with default
(if-let [default-val (get-default-value attr)]
(assoc node attr default-val)
;; If no default, remove the attribute
(dissoc node attr))
node))
node
text-attrs)))
(fix-position-data [position-data]
(mapv fix-text-attrs position-data))
(fix-text-content [content]
(types.text/transform-nodes types.text/is-content-node? fix-text-attrs content))
(update-shape [object]
(if (cfh/text-shape? object)
(-> object
(d/update-when :content fix-text-content)
(d/update-when :position-data fix-position-data))
object))
(update-container [container]
(d/update-when container :objects d/update-vals update-shape))]
(-> data
(update :pages-index d/update-vals update-container)
(d/update-when :components d/update-vals update-container))))
(defmethod migrate-data "0015-clean-shadow-color"
[data _]
(let [decode-shadow-color
@@ -1637,6 +1704,68 @@
(update :pages-index d/update-vals update-container)
(d/update-when :components d/update-vals update-container))))
;; Copy fills from position-data to text nodes when all text nodes lack fills,
;; all position-data have fills, and the counts match
(defmethod migrate-data "0016-copy-fills-from-position-data-to-text-node"
[data _]
(letfn [(get-text-nodes [content]
;; Get all leaf text nodes from the content tree
(when content
(->> (types.text/node-seq types.text/is-text-node? content)
(seq))))
(update-content [content fills-map]
;; Transform the content tree to update text nodes with their corresponding fills
;; fills-map is a map from text node to its fills
(types.text/transform-nodes
types.text/is-text-node?
(fn [text-node]
(if-let [fills (get fills-map text-node)]
(assoc text-node :fills fills)
text-node))
content))
(update-object [object]
(if (cfh/text-shape? object)
(let [content (:content object)
position-data (:position-data object)
text-nodes (get-text-nodes content)]
;; Check if conditions are met:
;; 1. Has at least one text node
;; 2. All text nodes have no fills or empty fills
;; 3. Has at least one position-data entry
;; 4. All position-data have fills
;; 5. The number of text nodes matches the number of position-data
(if (and (seq text-nodes)
(seq position-data)
(= (count text-nodes) (count position-data))
(every? (fn [text-node]
(let [fills (:fills text-node)]
(or (nil? fills) (empty? fills))))
text-nodes)
(every? (fn [pd]
(let [fills (:fills pd)]
(and (some? fills) (seq fills))))
position-data))
;; Apply the migration: create a map from each text node to its corresponding fills
(let [fills-map (zipmap text-nodes (map :fills position-data))]
(update object :content #(update-content % fills-map)))
;; Don't modify if conditions aren't met
object))
;; Not a text shape, return as-is
object))
(update-container [container]
(d/update-when container :objects d/update-vals update-object))]
(-> data
(update :pages-index d/update-vals update-container)
(d/update-when :components d/update-vals update-container))))
(def available-migrations
(into (d/ordered-set)
["legacy-2"
@@ -1708,4 +1837,6 @@
"0013-clear-invalid-strokes-and-fills"
"0014-fix-tokens-lib-duplicate-ids"
"0014-clear-components-nil-objects"
"0015-clean-shadow-color"]))
"0015-fix-text-attrs-blank-strings"
"0015-clean-shadow-color"
"0016-copy-fills-from-position-data-to-text-node"]))

2
common/src/app/common/files/tokens.cljc
View File

@@ -44,7 +44,7 @@
(let [attr? (set attributes)]
(->> (remove (fn [[k v]]
(and (attr? k)
(= v (token-identifier token))))
(= v (or (token-identifier token) token))))
applied-tokens)
(into {}))))

17
common/src/app/common/files/variant.cljc
View File

@@ -10,16 +10,23 @@
[app.common.types.components-list :as ctcl]
[app.common.types.variant :as ctv]))
(defn find-variant-components
"Find a list of the components thet belongs to this variant-id"
[data objects variant-id]
([data variant-id]
(let [page-id (->> data
:components
vals
(filter #(= (:variant-id %) variant-id))
first
:main-instance-page)
objects (dm/get-in data [:pages-index page-id :objects])]
(find-variant-components data objects variant-id)))
([data objects variant-id]
;; We can't simply filter components, because we need to maintain the order
(->> (dm/get-in objects [variant-id :shapes])
(map #(dm/get-in objects [% :component-id]))
(map #(ctcl/get-component data % true))
reverse))
reverse)))
(defn extract-properties-names
[shape data]
@@ -28,7 +35,6 @@
:variant-properties
(map :name)))
(defn extract-properties-values
"Get a map of properties associated to their possible values"
[data objects variant-id]
@@ -50,7 +56,6 @@
(get :objects))]
(dm/get-in objects [variant-id :shapes]))))
(defn is-secondary-variant?
[component data]
(let [shapes (get-variant-mains component data)]

14
common/src/app/common/flags.cljc
View File

@@ -33,7 +33,9 @@
:login-with-ldap
;; Uses any generic authentication provider that implements OIDC protocol as credentials.
:login-with-oidc
;; Allows registration with Open ID
;; Enables custom SSO flow
:login-with-custom-sso
;; Allows registration with OIDC (takes effect only when general `registration` is disabled)
:oidc-registration
;; This logs to console the invitation tokens. It's useful in case the SMTP is not configured.
:log-invitation-tokens})
@@ -118,18 +120,15 @@
:terms-and-privacy-checkbox
;; Only for developtment.
:tiered-file-data-storage
:token-units
:token-base-font-size
:token-color
:token-typography-types
:token-typography-composite
:token-shadow
:transit-readable-response
:user-feedback
;; TODO: remove this flag.
:v2-migration
:webhooks
;; TODO: deprecate this flag and consolidate the code
:export-file-v3
:render-wasm-dpr
:hide-release-modal
:subscriptions
@@ -169,9 +168,8 @@
:enable-google-fonts-provider
:enable-component-thumbnails
:enable-render-wasm-dpr
:enable-token-units
:enable-token-typography-types
:enable-token-typography-composite
:enable-token-color
:enable-inspect-styles
:enable-feature-fdata-objects-map])
(defn parse

1
common/src/app/common/geom/shapes.cljc
View File

@@ -162,6 +162,7 @@
(dm/export gtr/inverse-transform-matrix)
(dm/export gtr/transform-rect)
(dm/export gtr/calculate-geometry)
(dm/export gtr/calculate-selrect)
(dm/export gtr/update-group-selrect)
(dm/export gtr/update-mask-selrect)
(dm/export gtr/apply-transform)

67
common/src/app/common/logic/libraries.cljc
View File

@@ -12,8 +12,11 @@
[app.common.files.changes-builder :as pcb]
[app.common.files.helpers :as cfh]
[app.common.files.variant :as cfv]
[app.common.geom.matrix :as gmt]
[app.common.geom.point :as gpt]
[app.common.geom.rect :as grc]
[app.common.geom.shapes :as gsh]
[app.common.geom.shapes.common :as gco]
[app.common.logging :as log]
[app.common.logic.shapes :as cls]
[app.common.logic.variant-properties :as clvp]
@@ -26,6 +29,7 @@
[app.common.types.library :as ctl]
[app.common.types.page :as ctp]
[app.common.types.pages-list :as ctpl]
[app.common.types.path.segment :as segment]
[app.common.types.shape :as cts]
[app.common.types.shape-tree :as ctst]
[app.common.types.shape.interactions :as ctsi]
@@ -1512,7 +1516,7 @@
:shapes [(:id shape)]
:index index-after
:ignore-touched true
:syncing true}))
:allow-altering-copies true}))
(update :undo-changes conj (make-change
container
{:type :mov-objects
@@ -1520,7 +1524,7 @@
:shapes [(:id shape)]
:index index-before
:ignore-touched true
:syncing true})))]
:allow-altering-copies true})))]
(if (and (ctk/touched-group? parent :shapes-group) omit-touched?)
changes
@@ -1876,6 +1880,44 @@
roperations'
uoperations')))))))
(defn- set-path-new-values
[current-shape prev-shape transform]
(let [new-content (segment/transform-content
(:content current-shape)
(gmt/transform-in (gpt/point 0 0) transform))
new-points (-> (segment/content->selrect new-content)
(grc/rect->points))
points-center (gco/points->center new-points)
new-selrect (gsh/calculate-selrect new-points points-center)
shape (assoc current-shape
:content new-content
:points new-points
:selrect new-selrect)
prev-center (segment/content-center (:content prev-shape))
delta (gpt/subtract points-center (first new-points))
new-pos (gpt/subtract prev-center delta)]
(gsh/absolute-move shape new-pos)))
(defn- switch-path-change-value
[prev-shape ;; The shape before the switch
current-shape ;; The shape after the switch (a clean copy)
ref-shape ;; The referenced shape on the main component
;; before the switch
attr]
(let [old-width (-> ref-shape :selrect :width)
new-width (-> prev-shape :selrect :width)
old-height (-> ref-shape :selrect :height)
new-height (-> prev-shape :selrect :height)
transform (-> (gpt/point (/ new-width old-width)
(/ new-height old-height))
(gmt/scale-matrix))
shape (set-path-new-values current-shape prev-shape transform)]
(get shape attr)))
(defn- switch-text-change-value
[prev-content ;; The :content of the text before the switch
@@ -1992,6 +2034,12 @@
;; If the values are already equal, don't copy them
(= (get previous-shape attr) (get current-shape attr))
;; If the value is the same as the origin, don't copy it
(= (get previous-shape attr) (get origin-ref-shape attr))
;; If the attr is not touched, don't copy it
(not (touched attr-group))
;; If both variants (origin and destiny) don't have the same value
;; for that attribute, don't copy it.
;; Exceptions: :points :selrect and :content can be different
@@ -2007,10 +2055,7 @@
(not= (get origin-ref-shape attr) (get current-shape attr)))
;; The :content attr cant't be copied to elements of different type
(and (= attr :content) (not= (:type previous-shape) (:type current-shape)))
;; If the attr is not touched, don't copy it
(not (touched attr-group)))
(and (= attr :content) (not= (:type previous-shape) (:type current-shape))))
;; On texts, both text (the actual letters)
;; and attrs (bold, font, etc) are in the same attr :content.
@@ -2024,6 +2069,10 @@
(= :content attr)
(touched attr-group))
path-change?
(and (= :path (:type current-shape))
(contains? #{:points :selrect :content} attr))
;; position-data is a special case because can be affected by :geometry-group and :content-group
;; so, if the position-data changes but the geometry is touched we need to reset the position-data
;; so it's calculated again
@@ -2052,6 +2101,12 @@
(:content origin-ref-shape)
touched)
path-change?
(switch-path-change-value previous-shape
current-shape
origin-ref-shape
attr)
:else
(get previous-shape attr)))

Some files were not shown because too many files have changed in this diff Show More