mirror of
https://github.com/penpot/penpot.git
synced 2026-09-08 11:54:36 -04:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
64fe8e57e3 | ||
|
|
a4b9ccee8b | ||
|
|
2101d657e5 | ||
|
|
a8a441b7ad |
No files matched your search
@@ -1,41 +0,0 @@
|
||||
def specs: [.. | objects | select(has("tests") and has("file"))];
|
||||
def dur: [.tests[].results[]?.duration // 0] | add;
|
||||
|
||||
specs as $s
|
||||
| ($s | map(select(any(.tests[]; .status == "unexpected")))) as $failed
|
||||
| ($s | map(select(any(.tests[]; .status == "flaky")))) as $flaky
|
||||
| ($s | map(select(any(.tests[]; .status == "skipped")))) as $skipped
|
||||
| ($s | length) as $total
|
||||
| ($s | map(dur) | add // 0 | . / 1000 | floor) as $cpu
|
||||
| (if ($failed | length) > 0 then "❌"
|
||||
elif ($flaky | length) > 0 then "⚠️"
|
||||
else "✅" end) as $icon
|
||||
|
||||
| "## \($icon) Integration tests\n\n"
|
||||
+ "| Total | Passed | Flaky | Failed | Skipped | Test time |\n"
|
||||
+ "|---|---|---|---|---|---|\n"
|
||||
+ "| \($total) | \($total - ($failed|length) - ($flaky|length) - ($skipped|length)) "
|
||||
+ "| \($flaky|length) | \($failed|length) | \($skipped|length) | \($cpu / 60 | floor)m |\n"
|
||||
|
||||
+ (if ($failed | length) > 0 then
|
||||
"\n### Failed\n\n"
|
||||
+ ($failed | map("- `\(.file):\(.line)` — \(.title)") | join("\n")) + "\n"
|
||||
else "" end)
|
||||
|
||||
+ (if ($flaky | length) > 0 then
|
||||
"\n### Flaky (passed on retry)\n\n"
|
||||
+ ($flaky
|
||||
| map({ t: "`\(.file):\(.line)` — \(.title)",
|
||||
r: ([.tests[].results[]? | select(.status == "failed")] | length) })
|
||||
| sort_by(-.r)
|
||||
| map("- \(.t) _(\(.r) \(if .r == 1 then "retry" else "retries" end))_")
|
||||
| join("\n")) + "\n"
|
||||
else "" end)
|
||||
|
||||
+ (if $total > 0 then
|
||||
"\n<details><summary>Slowest specs</summary>\n\n"
|
||||
+ ($s | map({ t: "`\(.file)` — \(.title)", d: (dur / 1000 | floor) })
|
||||
| sort_by(-.d) | .[0:5]
|
||||
| map("- \(.t) — \(.d)s") | join("\n"))
|
||||
+ "\n\n</details>\n"
|
||||
else "" end)
|
||||
@@ -17,19 +17,15 @@ on:
|
||||
required: true
|
||||
default: 'develop'
|
||||
|
||||
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
|
||||
# caller's workflow, which put this workflow and the other reusable one called
|
||||
# by the same caller into a single shared group, and left a manual dispatch of
|
||||
# the same ref in a group of its own, free to race on the same artifacts.
|
||||
concurrency:
|
||||
group: build-bundle-${{ inputs.gh_ref }}
|
||||
group: ${{ github.workflow }}-${{ inputs.gh_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── 1. Decide whether there is anything to build ───────────────────────
|
||||
check:
|
||||
name: Check current bundle
|
||||
runs-on: penpot-standar-runner
|
||||
runs-on: penpot-runner-01
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
gh_ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
@@ -79,7 +75,7 @@ jobs:
|
||||
# ── 2. Build and upload, only when needed ──────────────────────────────
|
||||
build:
|
||||
name: Build and Upload Penpot Bundle
|
||||
runs-on: penpot-standar-runner
|
||||
runs-on: penpot-runner-01
|
||||
timeout-minutes: 90
|
||||
needs: check
|
||||
if: needs.check.outputs.exists == 'false'
|
||||
@@ -120,7 +116,7 @@ jobs:
|
||||
# ── 3. Single failure notification for the whole workflow ─────────────
|
||||
notify:
|
||||
name: Notify failure
|
||||
runs-on: penpot-standar-runner
|
||||
runs-on: penpot-runner-01
|
||||
timeout-minutes: 5
|
||||
needs: [check, build]
|
||||
if: failure()
|
||||
|
||||
@@ -5,10 +5,6 @@ on:
|
||||
schedule:
|
||||
- cron: '16 5-20 * * 1-5'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
@@ -23,7 +19,7 @@ jobs:
|
||||
with:
|
||||
gh_ref: "develop"
|
||||
|
||||
build-docker-admin-console:
|
||||
build-admin-console-docker:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
|
||||
@@ -6,7 +6,7 @@ on:
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and push DevEnv Docker image
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
|
||||
steps:
|
||||
- name: Set common environment variables
|
||||
|
||||
@@ -16,12 +16,8 @@ on:
|
||||
required: true
|
||||
default: 'develop'
|
||||
|
||||
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
|
||||
# caller's workflow, which put this workflow and the other reusable one called
|
||||
# by the same caller into a single shared group, and left a manual dispatch of
|
||||
# the same ref in a group of its own, free to race on the same artifacts.
|
||||
concurrency:
|
||||
group: build-docker-${{ inputs.gh_ref }}
|
||||
group: ${{ github.workflow }}-${{ inputs.gh_ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
@@ -36,7 +32,7 @@ jobs:
|
||||
# ── 1. Resolve the build key and check the whole set at once ───────────
|
||||
prepare:
|
||||
name: Prepare
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
gh_ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
@@ -111,7 +107,7 @@ jobs:
|
||||
# ── 2. One build per image, in parallel, only when needed ──────────────
|
||||
build:
|
||||
name: Build ${{ matrix.image }}
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
timeout-minutes: 60
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.exists == 'false'
|
||||
@@ -224,7 +220,7 @@ jobs:
|
||||
# the S3 marker guarantees the branch tags were already moved.
|
||||
promote:
|
||||
name: Promote image set
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
timeout-minutes: 10
|
||||
needs: [prepare, build]
|
||||
|
||||
@@ -271,7 +267,7 @@ jobs:
|
||||
# ── 4. Single failure notification for the whole workflow ─────────────
|
||||
notify:
|
||||
name: Notify failure
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
timeout-minutes: 5
|
||||
needs: [prepare, build, promote]
|
||||
if: failure()
|
||||
|
||||
@@ -5,10 +5,6 @@ on:
|
||||
schedule:
|
||||
- cron: '36 5-20 * * 1-5'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
@@ -23,7 +19,7 @@ jobs:
|
||||
with:
|
||||
gh_ref: "staging"
|
||||
|
||||
build-docker-admin-console:
|
||||
build-admin-console-docker:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
|
||||
@@ -6,12 +6,6 @@ on:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
# Keyed by ref and never cancelling: pushing 2.17.2 shortly after 2.17.2-RC1
|
||||
# must not abort the release already in flight.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
@@ -26,18 +20,10 @@ jobs:
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
|
||||
build-docker-admin-console:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
|
||||
notify:
|
||||
name: Notifications
|
||||
runs-on: ubuntu-24.04
|
||||
needs:
|
||||
- build-docker
|
||||
- build-docker-admin-console
|
||||
needs: build-docker
|
||||
steps:
|
||||
- name: Notify Mattermost
|
||||
uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e # v2.1.0
|
||||
@@ -51,9 +37,7 @@ jobs:
|
||||
|
||||
publish-final-tag:
|
||||
if: ${{ !contains(github.ref_name, '-RC') && !contains(github.ref_name, '-alpha') && !contains(github.ref_name, '-beta') && contains(github.ref_name, '.') }}
|
||||
needs:
|
||||
- build-docker
|
||||
- build-docker-admin-console
|
||||
needs: build-docker
|
||||
uses: ./.github/workflows/release.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
name: _TMP TOKENS
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '46 5-20 * * 1-5'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
gh_ref: "hiru-tokens-in-libs"
|
||||
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
gh_ref: "hiru-tokens-in-libs"
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: penpot-standar-runner
|
||||
runs-on: penpot-runner-01
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
test-backend:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Backend Tests"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
test-common:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Common Tests"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
composable-test-suite:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Run composable test suite (mocked backend)"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
name: "CI: Exporter"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'exporter/**'
|
||||
- 'common/**'
|
||||
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- ready_for_review
|
||||
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- staging
|
||||
|
||||
paths:
|
||||
- 'exporter/**'
|
||||
- 'common/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-exporter:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Exporter Tests"
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
- /var/cache/github-runner/m2:/root/.m2
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Lint
|
||||
working-directory: ./exporter
|
||||
run: |
|
||||
corepack enable;
|
||||
corepack install;
|
||||
pnpm install;
|
||||
pnpm run check-fmt:clj
|
||||
pnpm run lint:clj
|
||||
|
||||
- name: Tests
|
||||
working-directory: ./exporter
|
||||
run: |
|
||||
./scripts/test
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
test-frontend:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Frontend Tests"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -5,37 +5,11 @@ defaults:
|
||||
shell: bash
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
gh_ref:
|
||||
description: 'Name of the branch or ref'
|
||||
type: string
|
||||
required: true
|
||||
default: 'develop'
|
||||
|
||||
shards:
|
||||
description: 'Shard layout (JSON array)'
|
||||
type: choice
|
||||
required: true
|
||||
default: '[1, 2, 3, 4]'
|
||||
options:
|
||||
- '[1, 2, 3, 4]'
|
||||
- '[1, 2, 3, 4, 5, 6]'
|
||||
- '[1, 2]'
|
||||
- '[1]'
|
||||
|
||||
workers:
|
||||
description: 'Playwright workers per shard'
|
||||
type: string
|
||||
required: true
|
||||
default: '2'
|
||||
|
||||
pull_request:
|
||||
paths:
|
||||
- 'frontend/**'
|
||||
- 'common/**'
|
||||
- 'render-wasm/**'
|
||||
- '.github/workflows/tests-integration.yml'
|
||||
|
||||
types:
|
||||
- opened
|
||||
@@ -51,41 +25,25 @@ on:
|
||||
- 'frontend/**'
|
||||
- 'common/**'
|
||||
- 'render-wasm/**'
|
||||
- '.github/workflows/tests-integration.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.gh_ref || github.ref }}
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-integration:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Build Integration Bundle"
|
||||
runs-on: penpot-extended-runner
|
||||
timeout-minutes: 30
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
- /var/cache/github-runner/m2:/root/.m2
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
outputs:
|
||||
bundle_key: ${{ steps.vars.outputs.bundle_key }}
|
||||
|
||||
steps:
|
||||
# An empty `ref` makes checkout fall back to its default (the PR merge
|
||||
# ref on pull_request, the pushed ref on push).
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
# The cache key must come from the SHA actually checked out: on a manual
|
||||
# run `github.sha` points at the dispatching ref, not at `gh_ref`.
|
||||
- name: Extract cache key
|
||||
id: vars
|
||||
run: |
|
||||
echo "bundle_key=integration-bundle-$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build Bundle
|
||||
working-directory: ./frontend
|
||||
@@ -95,151 +53,41 @@ jobs:
|
||||
- name: Store Bundle Cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
key: ${{ steps.vars.outputs.bundle_key }}
|
||||
key: "integration-bundle-${{ github.sha }}"
|
||||
path: frontend/resources/public
|
||||
|
||||
test-integration:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Integration Tests (${{ matrix.shard }})"
|
||||
runs-on: penpot-extended-runner
|
||||
timeout-minutes: ${{ github.base_ref == 'staging' && 60 || 25 }}
|
||||
|
||||
needs: build-integration
|
||||
|
||||
# TEMPORARY (release stabilization): PRs targeting `staging` run on a
|
||||
# single serial shard, so new flakes cannot block the release work.
|
||||
# Remove the `github.base_ref` branch below to restore full parallelism.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: ${{ fromJSON(inputs.shards || (github.base_ref == 'staging' && '[1]' || '[1, 2, 3, 4]')) }}
|
||||
|
||||
name: "Integration Tests"
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
- /var/cache/github-runner/m2:/root/.m2
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
- /var/cache/github-runner/ms-playwright:/ms-playwright
|
||||
env:
|
||||
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
|
||||
|
||||
needs: build-integration
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
- name: Restore Cache
|
||||
uses: actions/cache/restore@v5
|
||||
with:
|
||||
key: ${{ needs.build-integration.outputs.bundle_key }}
|
||||
key: "integration-bundle-${{ github.sha }}"
|
||||
path: frontend/resources/public
|
||||
|
||||
- name: Install deps
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
corepack enable;
|
||||
corepack install;
|
||||
pnpm install --frozen-lockfile;
|
||||
|
||||
# No-op once the shared volume is warm; keeps the first run working.
|
||||
- name: Install Playwright Chromium
|
||||
working-directory: ./frontend
|
||||
run: pnpm exec playwright install chromium
|
||||
|
||||
# `strategy.job-total` is the matrix size, so the shard denominator
|
||||
# follows the `shards` input without being hardcoded.
|
||||
- name: Run Tests
|
||||
working-directory: ./frontend
|
||||
env:
|
||||
WORKERS: ${{ inputs.workers }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
run: |
|
||||
# TEMPORARY (release stabilization): see the note on the matrix above.
|
||||
if [ -z "$WORKERS" ]; then
|
||||
if [ "$BASE_REF" = "staging" ]; then WORKERS=1; else WORKERS=2; fi
|
||||
fi
|
||||
echo "Running shard ${{ matrix.shard }}/${{ strategy.job-total }} with $WORKERS workers"
|
||||
pnpm exec playwright test --project default \
|
||||
--workers="$WORKERS" \
|
||||
--shard=${{ matrix.shard }}/${{ strategy.job-total }} \
|
||||
--reporter=blob
|
||||
|
||||
- name: Upload blob report
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: integration-blob-report-${{ matrix.shard }}
|
||||
path: frontend/blob-report/
|
||||
overwrite: true
|
||||
retention-days: 3
|
||||
./scripts/test-e2e
|
||||
|
||||
- name: Upload test result
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: integration-tests-result-${{ matrix.shard }}
|
||||
name: integration-tests-result
|
||||
path: frontend/test-results/
|
||||
overwrite: true
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
merge-reports:
|
||||
if: ${{ always() && !github.event.pull_request.draft && needs.test-integration.result != 'skipped' }}
|
||||
name: "Merge Integration Reports"
|
||||
runs-on: penpot-extended-runner
|
||||
timeout-minutes: 15
|
||||
|
||||
needs: test-integration
|
||||
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
- /var/cache/github-runner/m2:/root/.m2
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
- name: Install deps
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
corepack enable;
|
||||
corepack install;
|
||||
pnpm install --frozen-lockfile;
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
path: frontend/all-blob-reports
|
||||
pattern: integration-blob-report-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge into HTML report
|
||||
working-directory: ./frontend
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_NAME: report.json
|
||||
run: |
|
||||
pnpm exec playwright merge-reports \
|
||||
--reporter=html,json,list ./all-blob-reports
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
if [ ! -f report.json ]; then
|
||||
echo "No report produced (all shards failed early)." >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
jq -r -f ../.github/scripts/playwright-summary.jq report.json >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload HTML report
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: integration-html-report
|
||||
path: frontend/playwright-report/
|
||||
overwrite: true
|
||||
retention-days: 7
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
test-library:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Library Tests"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: "CI: MCP"
|
||||
name: "MCP CI"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
test-mcp:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Test MCP"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container: penpotapp/devenv:latest
|
||||
|
||||
steps:
|
||||
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
api-test-suite-mocked:
|
||||
if: ${{ github.event_name != 'workflow_dispatch' && !github.event.pull_request.draft }}
|
||||
name: "Run Plugin API Test Suite (mocked)"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
# api-test-suite-live:
|
||||
# if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
# name: Run Plugin API Test Suite (live)
|
||||
# runs-on: penpot-extended-runner
|
||||
# runs-on: penpot-runner-02
|
||||
# container:
|
||||
# image: penpotapp/devenv:latest
|
||||
#
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
test-plugins:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: Plugins Runtime Linter & Tests
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
test-render-wasm:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Render WASM Tests"
|
||||
runs-on: penpot-extended-runner
|
||||
runs-on: penpot-runner-02
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
volumes:
|
||||
|
||||
@@ -58,8 +58,6 @@ opencode.json
|
||||
/docker/images/bundle*
|
||||
/exporter/target
|
||||
/exporter/.shadow-cljs
|
||||
/exporter/resources/wasm/
|
||||
/exporter/src/app/wasm/shared.js
|
||||
/frontend/.storybook/preview-body.html
|
||||
/frontend/.storybook/preview-head.html
|
||||
/frontend/playwright-report/
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
---
|
||||
description: Resolve local git conflicts and stage the resolved files with git add — never continues the rebase
|
||||
agent: build
|
||||
---
|
||||
|
||||
# Fix Git Conflicts
|
||||
|
||||
Resolve conflicts in the local repository. The user handles finishing the
|
||||
rebase themselves — you must **never** run `git rebase --continue`,
|
||||
`git rebase --skip`, `git merge --continue`, or anything similar.
|
||||
|
||||
## Phase 1 — Understand the problem (read-only)
|
||||
|
||||
1. Run `git status` to detect the conflict state (rebase, merge, cherry-pick, etc.) and list conflicted files.
|
||||
2. For each conflicted (unmerged) file, understand the situation **without modifying anything**:
|
||||
- Read the file and identify the conflict markers (`<<<<<<<`, `=======`, `>>>>>>>`).
|
||||
- Inspect both sides — `git show <ours>:<file>` and `git show <theirs>:<file>` — plus `git log`/`git show` on the commits involved to understand intent.
|
||||
- Identify what each side changed and why, and how they should be combined.
|
||||
|
||||
## Phase 2 — Present the resolution plan
|
||||
|
||||
3. **Present a clear plan to the user before touching any file.** For each conflicted file, state:
|
||||
- What each side changed and why.
|
||||
- Your proposed resolution and the reasoning behind it.
|
||||
- How the two sides are combined (both additive → merge; both modify the same code → keep the semantically correct version, merging intent from both sides when clear from code and context).
|
||||
4. **Ask the user only when genuinely unclear.** Do not ask about anything you can determine yourself from the code, commit messages, or context. Only decisions that are not determinable and change the outcome (e.g. conflicting product decisions, which side to discard) warrant a question. **Collect all such questions together in an "Open Questions" section at the end of the plan**, so the user has full context to answer them properly.
|
||||
5. **Wait for the user to accept the plan** (and answer any open questions) before editing, staging, or otherwise modifying anything.
|
||||
|
||||
## Phase 3 — Execute
|
||||
|
||||
6. Resolve each conflicted file by editing the file to the agreed merged content and removing all conflict markers.
|
||||
|
||||
## Phase 4 — Stage and verify
|
||||
|
||||
7. **Stage every resolved file** with `git add <file>`. Do not stage unrelated untracked files unless clearly part of the resolution.
|
||||
8. Verify no conflict markers remain (search for `<<<<<<<` / `>>>>>>>` in resolved files) and that `git status` shows no unmerged paths.
|
||||
|
||||
## Phase 5 — Report
|
||||
|
||||
9. Briefly report the conflict state, how each conflicted file was resolved (and any answers received to open questions), and stop — do **not** run `git rebase --continue` or any other continuation command.
|
||||
@@ -1,16 +1,34 @@
|
||||
Act as a senior software engineer and perform a thorough review.
|
||||
Act as a senior software engineer and perform a thorough code review.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. **Determine what is being reviewed** from the provided context:
|
||||
- **If it is a plan** (implementation plan, design document, task breakdown) → load the **`plan-review`** skill.
|
||||
- **If it is code** (diff, PR, code change) → load the **`code-review`** skill.
|
||||
|
||||
2. Read `AGENTS.md` and follow its instructions for finding and reading all related testing documentation from memories before reviewing.
|
||||
|
||||
1. Load the **`code-review-and-quality`** skill — it defines the five axes, core principles (DRY, KISS, YAGNI), severity taxonomy, and output format.
|
||||
2. Determine the diff or code to review from the provided context.
|
||||
3. **Skip generated files, lockfile-only changes, and unrelated modifications** unless they introduce security risks.
|
||||
|
||||
4. Follow the loaded skill's process and produce its output format.
|
||||
4. Read the diff and the surrounding context for each changed file.
|
||||
5. Review across all five axes: correctness, readability, architecture, security, performance.
|
||||
6. Produce the review using this structure:
|
||||
- **Summary**: One-paragraph overview of the change and its impact
|
||||
- **Critical/High Findings**: Blockers that must be fixed (with file:line, severity, description, and proposed fix)
|
||||
- **Other Findings**: Medium/Low issues and suggestions
|
||||
- **Testing Recommendations**: Missing test coverage or test quality issues
|
||||
- **Positive Observations**: What was done well (brief, specific)
|
||||
- **Verdict**: Approve / Request Changes / Needs Discussion
|
||||
7. For each finding:
|
||||
- State the severity (Critical / High / Medium / Low / Suggestion)
|
||||
- Identify the file and line
|
||||
- Describe failure circumstances
|
||||
- **For Critical/High**: Provide a concrete fix with a code snippet showing the corrected code
|
||||
- **For Medium/Low**: Describe the fix clearly; code snippet optional
|
||||
- If multiple approaches exist, briefly note trade-offs
|
||||
8. **Perform a second review pass if the change is complex:**
|
||||
- **Complex indicators**: Critical/High findings, multiple files (>5), architectural changes, security-sensitive code, >300 lines changed
|
||||
- **Skip for simple changes**: Typo fixes, formatting, small bug fixes (<50 lines), single-file changes with no findings
|
||||
- Second pass checks:
|
||||
- Validate severity assignments: Are Critical/High findings truly blockers?
|
||||
- Catch missed issues: Edge cases, error paths, test gaps overlooked in first pass
|
||||
- Remove false positives: Discard findings that aren't real issues
|
||||
- Verify fixes: Are the proposed solutions actually correct and complete?
|
||||
|
||||
## Strong Rules
|
||||
|
||||
@@ -18,8 +36,44 @@ Act as a senior software engineer and perform a thorough review.
|
||||
2. Do not modify any code and do not create a commit — this command only reviews.
|
||||
3. Be specific and constructive. "This could be better" is not helpful — explain why and how.
|
||||
4. Prioritize by impact. One structural issue outweighs ten nits.
|
||||
5. Missing tests are an issue, not a suggestion. Report as a severity-tagged finding — never as a recommendation.
|
||||
5. If tests are missing for new functionality, flag it as High severity.
|
||||
|
||||
## Context
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Expected Format
|
||||
|
||||
```
|
||||
## Review Summary
|
||||
[1-2 sentences on what the change does and overall assessment]
|
||||
|
||||
## Critical/High Findings
|
||||
### [Severity] file.ts:123
|
||||
**Issue**: [Description of the problem]
|
||||
**Impact**: [What could go wrong]
|
||||
**Fix**:
|
||||
```[language]
|
||||
// Current code
|
||||
[problematic code]
|
||||
|
||||
// Fixed code
|
||||
[corrected code]
|
||||
```
|
||||
[Optional: note trade-offs if multiple approaches exist]
|
||||
|
||||
## Other Findings
|
||||
### [Severity] file.ts:456
|
||||
**Issue**: [Description]
|
||||
**Fix**: [Clear description; code snippet optional]
|
||||
|
||||
## Testing Recommendations
|
||||
[List specific test cases that should be added]
|
||||
|
||||
## Positive Observations
|
||||
[2-3 specific things done well]
|
||||
|
||||
## Verdict
|
||||
[Approve / Request Changes / Needs Discussion]
|
||||
[If Request Changes: list the must-fix items]
|
||||
```
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
name: code-review
|
||||
name: code-review-and-quality
|
||||
description: Conducts multi-axis code review. Use before merging any change. Use when reviewing code written by yourself, another agent, or a human. Use when you need to assess code quality across multiple dimensions before it enters the main branch.
|
||||
---
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
---
|
||||
name: plan-review
|
||||
description: Reviews implementation plans for quality, completeness, and actionability. Use after a plan is produced by the planner skill, before starting implementation. Use when evaluating a plan written by yourself, another agent, or a human.
|
||||
---
|
||||
|
||||
# Plan Review
|
||||
|
||||
## Overview
|
||||
|
||||
Multi-dimensional plan review with quality gates. Every plan gets reviewed before implementation starts — no exceptions. Review covers six axes: completeness, task quality, architecture & sequencing, risk coverage, actionability, and proposed code quality.
|
||||
|
||||
**The approval standard:** Approve a plan when it is specific enough that a skilled implementer could execute it without guessing, the task ordering is sound, and risks are acknowledged. Perfect plans don't exist — the goal is confidence that implementation won't derail. Don't block a plan because it isn't exactly how you would have structured it. If it's executable and well-organized, approve it.
|
||||
|
||||
## When to Use
|
||||
|
||||
- After the planner skill produces a plan
|
||||
- Before starting implementation on any non-trivial task
|
||||
- When reviewing a plan written by another agent or a human
|
||||
- When a plan feels too large, vague, or risky to start
|
||||
|
||||
**Do NOT use for:** Single-file changes with obvious scope, or when the task is trivial enough to just do.
|
||||
|
||||
## The Six-Axis Review
|
||||
|
||||
Every plan gets evaluated across these dimensions:
|
||||
|
||||
### 1. Completeness
|
||||
|
||||
Does the plan cover everything needed to implement successfully?
|
||||
|
||||
- Is the **context** clear? (What problem, why now, what's the goal?)
|
||||
- Are **affected modules** identified with paths?
|
||||
- Are **architecture decisions** documented with rationale?
|
||||
- Is there a **testing strategy**?
|
||||
- Are **verification commands** explicit (not "run the tests")?
|
||||
- Are **open questions** listed (not buried in someone's head)?
|
||||
- Is there a **parallelization** assessment for multi-task plans?
|
||||
|
||||
**Missing any of these is a gap, not a nit.**
|
||||
|
||||
### 2. Task Quality
|
||||
|
||||
Are the tasks well-defined and independently executable?
|
||||
|
||||
- Does every task have **acceptance criteria**? (Testable, not vague)
|
||||
- Does every task have **verification steps**?
|
||||
- Are tasks **sized appropriately**? (XS–M is ideal, L is acceptable, XL must be split)
|
||||
- Are **dependencies** between tasks explicitly stated?
|
||||
- Are **files likely touched** listed?
|
||||
- Is each task a **single, self-contained change**? (Not "implement the whole feature")
|
||||
- Could a skilled implementer pick up any task and execute it without asking clarifying questions?
|
||||
|
||||
### 3. Architecture & Sequencing
|
||||
|
||||
Is the plan structured so implementation flows correctly?
|
||||
|
||||
- Does implementation order follow the **dependency graph** (foundations first)?
|
||||
- Are tasks **vertically sliced** (feature paths) rather than horizontally layered?
|
||||
- Does each task leave the system in a **working state**?
|
||||
- Are there **checkpoints** between major phases?
|
||||
- Are **high-risk tasks early** (fail fast)?
|
||||
- Is the total plan a reasonable number of tasks? (More than ~15 tasks suggests the scope should be split into multiple plans)
|
||||
|
||||
### 4. Risk Coverage
|
||||
|
||||
Are the hard parts acknowledged and mitigated?
|
||||
|
||||
- Are **edge cases** identified?
|
||||
- Are **breaking changes** or **migration concerns** noted?
|
||||
- Are **security implications** considered?
|
||||
- Are **performance implications** considered?
|
||||
- Are **external dependencies** or integration risks flagged?
|
||||
- Is there a plan for **rollback** if something goes wrong?
|
||||
- Are **data integrity** risks addressed (what happens if a migration fails mid-way)?
|
||||
|
||||
### 5. Actionability
|
||||
|
||||
Can an implementer actually execute this?
|
||||
|
||||
- Are **file paths** specific (not "update the relevant files")?
|
||||
- Are **function/method names** mentioned where applicable?
|
||||
- Are **verification commands** copy-pasteable (not "run the linter")?
|
||||
- Are **test commands** project-specific (not generic)?
|
||||
- Is the **code shape** described where the implementation isn't obvious?
|
||||
- Are **conventions** referenced (naming, patterns, existing utilities to reuse)?
|
||||
- Does the plan reference **existing code** the implementer should read first?
|
||||
|
||||
### 6. Proposed Code Quality *(when the plan includes implementation details)*
|
||||
|
||||
If the plan proposes code shapes, function signatures, data structures, or API designs, evaluate those proposals against `code-review` criteria:
|
||||
|
||||
- **Correctness:** Do the proposed types/signatures handle edge cases (null, empty, boundaries)?
|
||||
- **Readability:** Are proposed names descriptive and consistent with project conventions?
|
||||
- **Architecture:** Do proposed abstractions follow existing patterns? Are they justified (not over-engineered)?
|
||||
- **Security:** Do proposed APIs validate input at boundaries? Any injection/XSS vectors in the design?
|
||||
- **Performance:** Do proposed data structures avoid N+1 patterns? Any unbounded operations in the design?
|
||||
|
||||
**When to apply:** Only when the plan includes specific code snippets, type definitions, API contracts, or function signatures. Plans that only describe "what" without showing "how" skip this axis.
|
||||
|
||||
## Structural Remedies
|
||||
|
||||
When you flag a structural problem in a plan, propose the fix — not just the problem:
|
||||
|
||||
- **A task is too large (XL):** Split it into vertical slices. Each slice should be independently testable.
|
||||
- **Missing acceptance criteria:** Draft 2–3 specific, testable conditions for the task.
|
||||
- **Wrong sequencing:** Identify the dependency and propose the correct order.
|
||||
- **No checkpoints:** Suggest where checkpoints should go (typically after every 2–3 tasks).
|
||||
- **Vague verification:** Replace "run tests" with the actual project command.
|
||||
- **Horizontal slicing:** Restructure into vertical feature paths.
|
||||
- **Missing risk section:** Draft the risks you can identify from the plan content.
|
||||
|
||||
Prefer the remedy that makes the plan immediately actionable over one that just flags the gap.
|
||||
|
||||
## Plan Sizing
|
||||
|
||||
Plans should be scoped to a single deliverable:
|
||||
|
||||
```
|
||||
1–5 tasks → Good. A focused feature or bug fix.
|
||||
6–10 tasks → Acceptable for a moderate feature.
|
||||
11–15 tasks → Large. Consider splitting into phases.
|
||||
15+ tasks → Too large. Split into multiple plans.
|
||||
```
|
||||
|
||||
**What counts as "one plan":** A self-contained set of changes that delivers a single coherent capability. If you can describe the goal in one sentence, it's one plan.
|
||||
|
||||
## Categorize Findings
|
||||
|
||||
Label every comment with its severity so the author knows what's required vs optional:
|
||||
|
||||
| Prefix | Meaning | Author Action |
|
||||
|--------|---------|---------------|
|
||||
| *(no prefix)* | Required change | Must address before implementation starts |
|
||||
| **Critical:** | Blocks implementation | Missing security consideration, data integrity risk, fundamentally wrong approach |
|
||||
| **Nit:** | Minor, optional | Author may ignore — wording, formatting |
|
||||
| **Optional:** / **Consider:** | Suggestion | Worth considering but not required |
|
||||
| **FYI** | Informational only | No action needed — context for future reference |
|
||||
|
||||
**Lead with what matters.** Order findings by leverage: missing risks and wrong sequencing first, then task quality gaps, then completeness, then nits. If you have one critical sequencing problem and ten nits, the sequencing problem *is* the review.
|
||||
|
||||
## Review Process
|
||||
|
||||
### Step 1: Understand the Goal
|
||||
|
||||
Before evaluating structure, understand intent:
|
||||
|
||||
```
|
||||
- What is this plan trying to accomplish?
|
||||
- What problem does it solve?
|
||||
- What does "done" look like?
|
||||
```
|
||||
|
||||
### Step 2: Check Completeness First
|
||||
|
||||
Scan for missing sections before diving into content:
|
||||
|
||||
```
|
||||
- Context present?
|
||||
- Affected modules listed?
|
||||
- Architecture decisions documented?
|
||||
- Risks acknowledged?
|
||||
- Testing strategy defined?
|
||||
- Verification commands explicit?
|
||||
```
|
||||
|
||||
### Step 3: Review Task Quality
|
||||
|
||||
Walk through each task:
|
||||
|
||||
```
|
||||
For each task:
|
||||
1. Can I tell exactly what to build?
|
||||
2. Are acceptance criteria specific and testable?
|
||||
3. Is the size reasonable (not XL)?
|
||||
4. Are dependencies clear?
|
||||
5. Would I know which files to touch?
|
||||
```
|
||||
|
||||
### Step 4: Validate Sequencing
|
||||
|
||||
Check the dependency graph:
|
||||
|
||||
```
|
||||
- Are foundations built first?
|
||||
- Does each task leave the system working?
|
||||
- Are checkpoints placed correctly?
|
||||
- Are high-risk items early?
|
||||
- Is it vertically sliced?
|
||||
```
|
||||
|
||||
### Step 5: Assess Actionability
|
||||
|
||||
Put yourself in the implementer's shoes:
|
||||
|
||||
```
|
||||
- Could I pick up task 1 and start coding without asking any questions?
|
||||
- Are the verification commands copy-pasteable?
|
||||
- Are file paths and function names specific?
|
||||
- Is existing code referenced where I'd need to read it?
|
||||
```
|
||||
|
||||
### Step 6: Verify the Verification Story
|
||||
|
||||
Check that the plan can actually confirm it worked:
|
||||
|
||||
```
|
||||
- What tests should pass after implementation?
|
||||
- What build/compile commands are relevant?
|
||||
- What manual checks are needed?
|
||||
- How do we know the feature works end-to-end?
|
||||
```
|
||||
|
||||
### Step 7: Evaluate Proposed Code Quality *(if applicable)*
|
||||
|
||||
If the plan includes code snippets, types, or API designs:
|
||||
|
||||
```
|
||||
- Load code-review skill for criteria
|
||||
- Check proposed signatures for edge cases
|
||||
- Verify naming follows project conventions
|
||||
- Confirm abstractions follow existing patterns
|
||||
- Scan for security vectors in proposed APIs
|
||||
- Check for performance issues in proposed data structures
|
||||
```
|
||||
|
||||
## Review Checklist
|
||||
|
||||
```markdown
|
||||
## Review: [Plan title]
|
||||
|
||||
### Completeness
|
||||
- [ ] Context explains the problem and goal
|
||||
- [ ] Affected modules are listed with paths
|
||||
- [ ] Architecture decisions have rationale
|
||||
- [ ] Testing strategy is defined
|
||||
- [ ] Verification commands are explicit and project-specific
|
||||
- [ ] Open questions are listed
|
||||
|
||||
### Task Quality
|
||||
- [ ] Every task has acceptance criteria
|
||||
- [ ] Every task has verification steps
|
||||
- [ ] Tasks are sized XS–M (L acceptable, XL must be split)
|
||||
- [ ] Task dependencies are stated
|
||||
- [ ] Files likely touched are listed
|
||||
|
||||
### Architecture & Sequencing
|
||||
- [ ] Order follows dependency graph (foundations first)
|
||||
- [ ] Vertically sliced (not horizontal layers)
|
||||
- [ ] Each task leaves system working
|
||||
- [ ] Checkpoints exist between phases
|
||||
- [ ] High-risk tasks are early
|
||||
|
||||
### Risk Coverage
|
||||
- [ ] Edge cases identified
|
||||
- [ ] Breaking changes / migrations noted
|
||||
- [ ] Security implications considered
|
||||
- [ ] Performance implications considered
|
||||
- [ ] Rollback strategy exists (if applicable)
|
||||
|
||||
### Actionability
|
||||
- [ ] File paths are specific
|
||||
- [ ] Verification commands are copy-pasteable
|
||||
- [ ] Existing code to read is referenced
|
||||
- [ ] Conventions and patterns are noted
|
||||
|
||||
### Proposed Code Quality *(if plan includes implementation details)*
|
||||
- [ ] Proposed types/signatures handle edge cases
|
||||
- [ ] Proposed names follow project conventions
|
||||
- [ ] Proposed abstractions follow existing patterns
|
||||
- [ ] No security vectors in proposed APIs
|
||||
- [ ] No performance issues in proposed structures
|
||||
|
||||
### Verdict
|
||||
- [ ] **Approve** — Ready to implement
|
||||
- [ ] **Request changes** — Gaps must be addressed
|
||||
```
|
||||
|
||||
## Common Rationalizations
|
||||
|
||||
| Rationalization | Reality |
|
||||
|---|---|
|
||||
| "I'll figure out the details during implementation" | That's how you discover blocking dependencies mid-task. Surface them now. |
|
||||
| "The tasks are obvious, no need for criteria" | Write them anyway. Explicit criteria surface hidden assumptions. |
|
||||
| "It's just a small feature, it doesn't need a plan" | Small features have edge cases too. 3 tasks with criteria takes 5 minutes. |
|
||||
| "The plan is good enough" | "Good enough" without acceptance criteria means the implementer defines "done" — and they might define it differently. |
|
||||
| "I'll add verification steps later" | Later never comes. The plan is the contract — define verification now. |
|
||||
| "Risks are minimal" | Every change has risks. If you can't name them, you haven't thought about them. |
|
||||
| "The file paths are obvious" | They're obvious to the author. The implementer might not know the codebase. |
|
||||
| "The code in the plan is fine, it'll get reviewed later" | Plan-level code review catches design problems before implementation — fixing them after coding is more expensive. |
|
||||
|
||||
## Red Flags
|
||||
|
||||
- No acceptance criteria on any task
|
||||
- Tasks that say "implement the feature" without specifics
|
||||
- No verification steps anywhere in the plan
|
||||
- All tasks are XL-sized
|
||||
- No checkpoints between phases
|
||||
- Dependency order isn't considered (e.g., API handler before domain model)
|
||||
- No testing strategy
|
||||
- Verification commands are generic ("run tests") instead of project-specific
|
||||
- Plan has 20+ tasks (scope too large for one plan)
|
||||
- No risk section on a plan with migrations, breaking changes, or security implications
|
||||
- Horizontal slicing (all domain, then all services, then all API)
|
||||
- File paths are vague ("update the relevant files")
|
||||
- Missing open questions section despite stated unknowns
|
||||
- Proposed code ignores project conventions or existing patterns
|
||||
- Proposed types use gratuitous `any`/`unknown`/optional without justification
|
||||
- Proposed APIs don't validate input at boundaries
|
||||
|
||||
## See Also
|
||||
|
||||
- For producing plans, use the `planner` skill
|
||||
- For reviewing implemented code, use `code-review` — also the criteria source for axis 6
|
||||
- For security-specific concerns, see `security-and-hardening`
|
||||
- For testing strategy guidance, see `testing`
|
||||
@@ -1,78 +0,0 @@
|
||||
---
|
||||
name: ste
|
||||
description: Write or rewrite text in ASD-STE100 Simplified Technical English. ONLY use this skill when the user explicitly invokes it by name — i.e. they type "/ste" or literally write "use the ste skill" / "apply ASD-STE100". Do NOT trigger it on paraphrased intent such as "simplify this", "make it clearer", "write technical documentation", or "shorter sentences please" — the user has deliberately scoped this skill to explicit invocation only. For those requests, respond normally without loading this skill unless they name it.
|
||||
---
|
||||
|
||||
# ASD-STE100 Simplified Technical English
|
||||
|
||||
Apply the ASD-STE100 standard to all prose you produce in this task. Do not announce that you use STE, do not name the standard, and do not explain the style unless the user asks. If the user later asks you to "write more naturally," ask one short question to confirm they want to leave STE before you drop it.
|
||||
|
||||
Compliance note (for you, not for output): the official specification and its dictionary are copyright ASD. This skill encodes paraphrased rules and a publicly sourced word list. For certified aerospace/defense deliverables, tell the user that full compliance requires the free official specification (asd-ste100.org) and a human sign-off. Never claim certified compliance.
|
||||
|
||||
## Step 0 — Classify the text
|
||||
|
||||
Before writing a single sentence, decide: is this **procedural** text (instructions someone follows) or **descriptive** text (explanation, background, description)? Every limit below depends on this. Mixed documents get classified section by section.
|
||||
|
||||
## Core rules
|
||||
|
||||
### Sentences
|
||||
- Procedural: maximum **20 words** per sentence.
|
||||
- Descriptive: maximum **25 words** per sentence.
|
||||
- Maximum **6 sentences** per paragraph. One topic per paragraph.
|
||||
- One instruction per sentence. Two actions in one sentence only if they occur at the same time.
|
||||
- Put a condition BEFORE its command: "If the pressure decreases, close the valve."
|
||||
- Do not omit articles, subjects, or verbs to save words. "Ensure file exists" is wrong; "Make sure that the file exists" is correct. Keep the word "that" after verbs like "make sure."
|
||||
- Numbers, units with numbers, abbreviations, quoted strings, code identifiers, and proper nouns each count as one word.
|
||||
|
||||
### Verbs
|
||||
- Allowed forms only: infinitive, imperative, simple present, simple past, simple future, and past participle used as an adjective.
|
||||
- Never use present perfect or continuous forms. "We have received" → "We received." "is being tested" → a simple form.
|
||||
- Never use an -ing form as a verb. An -ing word is allowed only inside a technical name ("the mounting bracket," "logging").
|
||||
- Active voice. Passive is allowed only in descriptive text when the agent is unknown or unimportant.
|
||||
- Instructions use the imperative: "Open the panel," not "You must open the panel" or "The panel should be opened."
|
||||
- Express actions as verbs, not nouns: "compress the file," not "perform compression of the file."
|
||||
- Modals: use **can** (possibility), **will** (future), **must** (requirement). Do not use should, would, could, may, might. A hedge becomes a fact or a "can": "an explosion can occur."
|
||||
- No phrasal verbs: "go down" → "decrease," "set up" → "install," "carry out" → "do."
|
||||
|
||||
### Words
|
||||
- One word, one meaning, one part of speech, used consistently. Never rotate synonyms: pick one name for a thing and repeat it.
|
||||
- Before drafting, replace unapproved vocabulary. Read `references/word-substitutions.md` and apply it; it is the working dictionary for this skill.
|
||||
- Domain-specific nouns (part names, tool names, product names, UI labels) and domain verbs (drill, ream, boot, compile) are your **technical nouns/verbs** — keep them as-is, use each consistently, and do not verb a noun or noun a verb.
|
||||
- Noun clusters: maximum **3 words** ("overhead panel light" is the limit). Longer clusters get decomposed with prepositions or hyphenated on first use: "main-gear-door retraction-winch handle."
|
||||
- American English spelling.
|
||||
- No Latin abbreviations: "e.g." → "for example," "i.e." → "that is," delete "etc."
|
||||
|
||||
### Punctuation
|
||||
- No semicolons — write two sentences.
|
||||
- Parentheses only for references, abbreviations, and item numbers.
|
||||
- Hyphenate words that act as one unit; a hyphenated word counts as one word.
|
||||
- No contractions.
|
||||
|
||||
### Warnings, cautions, notes
|
||||
- **WARNING** = risk of injury or death. **CAUTION** = risk of damage. **NOTE** = information only, never an instruction.
|
||||
- Start a warning or caution with the command or condition, then give the risk:
|
||||
"WARNING: Do not touch the terminal. The terminal has a dangerous voltage."
|
||||
- Notes obey the 25-word descriptive limit.
|
||||
|
||||
## Step 2 — Self-check pass
|
||||
|
||||
After drafting, scan your text once for each of these and fix every hit before you respond:
|
||||
|
||||
1. Any sentence over the 20/25-word limit for its type
|
||||
2. Contractions, semicolons
|
||||
3. "should," "would," "could," "may," "might"
|
||||
4. "has been," "have been," "had been," "is being," "was being"
|
||||
5. -ing words used as verbs
|
||||
6. Missing articles (a/an/the/this) before nouns
|
||||
7. Synonym rotation (the same object under two names)
|
||||
8. Any word in the unapproved column of `references/word-substitutions.md`
|
||||
9. Warnings that state the risk before the command
|
||||
|
||||
## Reference files
|
||||
|
||||
- `references/word-substitutions.md` — unapproved → approved word mappings and one-meaning rulings. Read it before drafting; it is short.
|
||||
- `references/examples.md` — worked before/after rewrites (procedural, descriptive, warnings, common mistakes). Read it when rewriting existing text or when unsure how a rule applies.
|
||||
|
||||
## What NOT to touch
|
||||
|
||||
Code blocks, command strings, file paths, error messages, quoted UI text, and proper nouns stay exactly as written. STE applies to the prose around them.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Worked before/after examples
|
||||
|
||||
## Verb forms
|
||||
|
||||
| Before | After |
|
||||
|---|---|
|
||||
| We have received the technical reports from HQ. | We received the technical reports from HQ. |
|
||||
| This device has been being used at Boeing since 2005. | Boeing started to use this device in 2005. |
|
||||
| The test is continued by the operator. | Continue the test. |
|
||||
| The screws should be replaced. | Replace the screws. |
|
||||
| The system is currently running diagnostics. | The system does diagnostic tests now. |
|
||||
|
||||
## Vocabulary and phrasing
|
||||
|
||||
| Before | After |
|
||||
|---|---|
|
||||
| Ensure file exists before running. | Make sure that the file exists before you run the command. |
|
||||
| Rotate the cover until the jacks are accessible. | Turn the cover until you can get access to the jacks. |
|
||||
| Extend the jack until the wheels are clear of the ground. | Extend the jack until the wheels do not touch the ground. |
|
||||
| Make sure that these steps are followed. | Obey these steps. |
|
||||
| Utilize approximately 3 liters of water. | Use about 3 liters of water. |
|
||||
| Perform a compression of the log files prior to upload. | Compress the log files before you upload them. |
|
||||
|
||||
## Noun clusters
|
||||
|
||||
| Before | After |
|
||||
|---|---|
|
||||
| Main gear door retraction winch handle | Main-gear-door retraction-winch handle |
|
||||
| Runway light connection resistance calibration | Calibration of the resistance of the runway light connection |
|
||||
| Remove the engine transmission housing attachment bolts. | Remove the bolts that attach the transmission housing to the engine. |
|
||||
|
||||
## Procedural rewrite (condition first, one instruction per sentence)
|
||||
|
||||
Before:
|
||||
> You should check that the reservoir has been filled with the correct fluid and, after attaching a clear tube to the bleed screw while ensuring its free end is submerged, the pedal can be pumped and held so that opening the screw releases trapped air.
|
||||
|
||||
After:
|
||||
> 1. Fill the reservoir with the correct fluid.
|
||||
> 2. Attach a clear tube to the bleed screw.
|
||||
> 3. Put the free end of the tube in a container of fluid.
|
||||
> 4. Push the pedal three times. Hold the pedal down.
|
||||
> 5. Open the bleed screw one half turn. Air and fluid flow into the tube.
|
||||
> 6. Close the bleed screw. Release the pedal.
|
||||
> 7. If air continues to come out, do steps 4 thru 6 again.
|
||||
|
||||
## Warnings and cautions (command first, then risk)
|
||||
|
||||
Before:
|
||||
> Note that serious data loss may potentially occur if the --force flag is used against production.
|
||||
|
||||
After:
|
||||
> CAUTION: Do not use the --force flag on the production database. The flag deletes the rows that do not match the source.
|
||||
|
||||
Before:
|
||||
> Touching the terminal could result in electrocution.
|
||||
|
||||
After:
|
||||
> WARNING: Do not touch the terminal. The terminal has a dangerous voltage.
|
||||
|
||||
## Common mistakes checklist
|
||||
|
||||
- Dropped articles: "Insert pin in bracket" → "Insert the pin in the bracket."
|
||||
- Synonym rotation: check/verify/confirm for the same action → one term, everywhere.
|
||||
- Hedges: "you may want to," "it is recommended that" → an imperative or "must."
|
||||
- Instruction buried in a NOTE: notes never instruct. Move the instruction to a numbered step.
|
||||
- Semicolon joining two clauses → two sentences.
|
||||
- "There are three bolts on the panel" → "The panel has three bolts."
|
||||
@@ -1,68 +0,0 @@
|
||||
# Word substitutions and one-meaning rulings
|
||||
|
||||
Compiled from public secondary sources (STEMG/ASD public pages, TechScribe, Acrolinx, training materials). This is a working approximation, not the official ASD dictionary. When a word is not listed here and feels formal or Latin-derived, prefer the shortest common alternative.
|
||||
|
||||
## Unapproved → approved
|
||||
|
||||
| Do not use | Use instead |
|
||||
|---|---|
|
||||
| utilize, leverage, employ | use |
|
||||
| commence, initiate, begin, originate | start |
|
||||
| terminate, cease, conclude | stop, end |
|
||||
| ensure, verify, confirm, validate, check | make sure (that), examine |
|
||||
| perform, conduct, execute, carry out | do |
|
||||
| facilitate, assist | help |
|
||||
| obtain, acquire, procure | get |
|
||||
| sufficient, adequate | enough |
|
||||
| approximately | about |
|
||||
| prior to | before |
|
||||
| subsequent to, following (prep.) | after |
|
||||
| adjacent to | near |
|
||||
| accomplish | do |
|
||||
| additional, supplementary | more |
|
||||
| attempt | try |
|
||||
| require, necessitate | need, must |
|
||||
| mandatory | necessary |
|
||||
| indicate, signify | show |
|
||||
| observe (=watch) | look at, examine |
|
||||
| rotate | turn |
|
||||
| deactivate | turn off, set to off |
|
||||
| activate, energize (unless technical verb) | turn on, start |
|
||||
| toxic | poisonous |
|
||||
| in order to | to |
|
||||
| via, by means of | through, with |
|
||||
| due to, owing to | because of |
|
||||
| in the event of/that | if |
|
||||
| accessible | (rewrite: "you can get access to") |
|
||||
| remainder | rest |
|
||||
| demonstrate | show |
|
||||
| modify, alter | change |
|
||||
| construct, fabricate, build | assemble, make |
|
||||
| retain | keep |
|
||||
| locate (=find) | find |
|
||||
| depress (a button) | push, press |
|
||||
| proceed | continue, go |
|
||||
|
||||
## One meaning, one part of speech (canonical rulings)
|
||||
|
||||
- **close** — verb only: to move to a position that stops flow, or to operate a circuit breaker. The adjective is unapproved → use **near** ("do not go near the propeller").
|
||||
- **test** — noun only: "do a test," never "test the system."
|
||||
- **check** — do not use as a verb for verification → "make sure that" or "examine."
|
||||
- **follow** — means only "come after." For rules and steps use **obey**: "Obey the safety instructions."
|
||||
- **fall** — means only "move down by gravity." For quantities use **decrease**. Never the season.
|
||||
- **oil** — noun only. "Oil the bearing" → "Put oil on the bearing" / "Lubricate the bearing."
|
||||
- **right** — direction only, never "correct."
|
||||
- **clear** — "without blockage." "Wheels are clear of the ground" → "wheels do not touch the ground."
|
||||
- **help** — verb only; the noun is **aid** ("with the aid of a mirror").
|
||||
- **above / below** — physical position only. For quantities: **more than / less than**.
|
||||
- **about** — two approved senses: "approximately" and "on the subject of." Use carefully.
|
||||
- **turn** — the general verb for rotation; "turn on / turn off" for power state is standard.
|
||||
- **level** — approved as noun and adjective (documented exception to the one-POS rule).
|
||||
|
||||
## Frequent-offender function words
|
||||
|
||||
- **should / would / could / may / might** — never. Requirement → **must**. Possibility → **can**. Future → **will**.
|
||||
- **etc.** — delete, or write the full list.
|
||||
- **e.g. / i.e.** — "for example" / "that is."
|
||||
- **any / appropriate / applicable / relevant** as hedges — replace with the specific thing meant.
|
||||
- **there is / there are** openers — rewrite with a real subject: "There are three bolts on the panel" → "The panel has three bolts."
|
||||
@@ -34,8 +34,7 @@ Before writing any test, read:
|
||||
2. Module-specific testing memory for the affected module:
|
||||
- `mem:common/testing` — CLJC unit tests
|
||||
- `mem:frontend/testing` — CLJS unit tests, Playwright E2E
|
||||
- `mem:backend/testing` — JVM clojure.test conventions
|
||||
- `mem:exporter/testing` — exporter unit tests
|
||||
- `mem:backend/core` — JVM clojure.test conventions
|
||||
|
||||
## Key Rules
|
||||
|
||||
|
||||
@@ -212,37 +212,6 @@ superseded it:
|
||||
|
||||
Replace the reference in the changelog entry with the correct merged PR number.
|
||||
|
||||
### 5b. Security advisory (GHSA) entries
|
||||
|
||||
Security advisories fixed in a release are documented in the changelog even
|
||||
though they are **neither milestone issues nor PRs**. The GHSA ID and its
|
||||
description are supplied by the user or the release notes — they never come
|
||||
from the milestone fetch in step 2.
|
||||
|
||||
**Format** (matches the existing precedent in `CHANGES.md`, e.g. the
|
||||
`create-font-variant` arbitrary file read advisory):
|
||||
|
||||
```markdown
|
||||
- Fix <user-facing description> (https://github.com/penpot/penpot/security/advisories/GHSA-XXXX-XXXX-XXXX)
|
||||
```
|
||||
|
||||
Rules:
|
||||
- Place the entry under `### :bug: Bugs fixed`, with **no issue or PR link** —
|
||||
only the advisory URL.
|
||||
- The advisory may be **draft/unpublished** at changelog time (the URL 404s
|
||||
publicly). Do **not** web-fetch or verify the URL, and do **not** drop the
|
||||
entry because of that. Rely on the GHSA ID provided by the user.
|
||||
- Derive the description from the supplied advisory title, imperative mood and
|
||||
user-facing (e.g. `Fix command injection in SVG exporter via legacy fill-color`).
|
||||
- These entries are **invisible to the automation**: they are not returned by
|
||||
`gh.py issues`, not matched by `--compare` (step 3), not part of the PR
|
||||
cross-reference (step 10), and not scanned by the anomaly-report regexes
|
||||
(step 11, which only match `issues/` and `pull/` links). Add them manually.
|
||||
- During pre-flight checks (step 6a) apply only the **backport/duplicate**
|
||||
check: if the same GHSA already appears in an earlier version section, remove
|
||||
it from the current section. Their absence from milestone cross-references
|
||||
is expected, not an anomaly.
|
||||
|
||||
### 6. Read the current CHANGES.md
|
||||
|
||||
Read the top of `CHANGES.md` to understand the existing format and find the
|
||||
@@ -431,8 +400,6 @@ if closed:
|
||||
- ✅ Every merged milestone PR is either in the changelog or excluded by label
|
||||
- ✅ PR and issue counts are internally consistent
|
||||
- ✅ No false-positive PR-to-issue associations
|
||||
- ✅ Advisory (GHSA) entries are not milestone PRs — their absence from the
|
||||
cross-reference is intentional (see step 5b)
|
||||
|
||||
## Version section template
|
||||
|
||||
@@ -443,12 +410,8 @@ if closed:
|
||||
|
||||
- <fix description> [#<ISSUE>](https://github.com/penpot/penpot/issues/<ISSUE>) (PR: [#<PR>](https://github.com/penpot/penpot/pull/<PR>))
|
||||
- <fix description> (by @contributor) [#<ISSUE>](https://github.com/penpot/penpot/issues/<ISSUE>) (PR: [#<PR>](https://github.com/penpot/penpot/pull/<PR>))
|
||||
- <fix description> (https://github.com/penpot/penpot/security/advisories/GHSA-XXXX-XXXX-XXXX)
|
||||
```
|
||||
|
||||
Advisory (GHSA) entries have no issue or PR link — just the advisory URL. See
|
||||
step 5b.
|
||||
|
||||
### 11. Generate anomaly report and save to CHANGES-ISSUES.md
|
||||
|
||||
After all edits and cross-referencing are complete, generate a structured
|
||||
@@ -769,14 +732,6 @@ self-contained and clickable in any Markdown viewer.
|
||||
Taiga description text or by searching GitHub PRs that reference the Taiga
|
||||
URL. Replace the Taiga reference with the GitHub issue link and add the PR
|
||||
reference if applicable.
|
||||
- **Security advisory (GHSA) entries.** Advisories fixed in the release are
|
||||
listed under `### :bug: Bugs fixed` with the advisory URL and **no issue or
|
||||
PR link**, even though they are not in the milestone. The GHSA ID and
|
||||
description come from the user — do **not** fetch or verify the URL, and do
|
||||
not drop a draft (unpublished) advisory. Precedent:
|
||||
`- Fix arbitrary file read security issue on create-font-variant rpc method
|
||||
(https://github.com/penpot/penpot/security/advisories/GHSA-xp3f-g8rq-9px2)`.
|
||||
See step 5b.
|
||||
- **Re-fetch before editing.** Milestones can change — always re-fetch issues
|
||||
before making edits, don't rely on cached data.
|
||||
- **Use `scripts/gh.py`.** Prefer the helper script over raw `gh api` calls for
|
||||
|
||||
@@ -5,8 +5,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m
|
||||
## Focused memories
|
||||
|
||||
- RPC, DB helpers, workers, cron: `mem:backend/rpc-db-worker-subtleties`
|
||||
- Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`.
|
||||
- HTTP sessions, config, media processing, and file data persistence: `mem:backend/http-storage-filedata-subtleties`.
|
||||
- HTTP sessions, config, storage, media, file data persistence: `mem:backend/http-storage-filedata-subtleties`
|
||||
- Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains`
|
||||
- Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`.
|
||||
|
||||
@@ -102,5 +101,10 @@ misleading linter/compiler output. See `mem:scripts/paren-repair`.
|
||||
|
||||
## Testing
|
||||
|
||||
Backend test commands, coverage rules, and conventions: `mem:backend/testing`.
|
||||
Cross-cutting testing principles, anti-patterns, and verification checklist: `mem:testing`.
|
||||
IMPORTANT: all CLI commands must be executed from the `backend/` subdirectory. JVM tests are invoked directly via `clojure -M:dev:test` — there is no pnpm wrapper. If you need to filter output, tee to a temp file first: `clojure -M:dev:test 2>&1 | tee /tmp/penpot-test-output.txt`. See `mem:testing` for execution discipline.
|
||||
|
||||
* **Coverage:** If code is added or modified in `src/`, corresponding tests in `test/backend_tests/` must be added or updated.
|
||||
* **Isolated run:** `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace.
|
||||
* **Regression run:** `clojure -M:dev:test` to ensure no regressions in related functional areas.
|
||||
* **Principles:** Cross-cutting testing principles, anti-patterns, and verification checklist: `mem:testing`.
|
||||
|
||||
@@ -14,7 +14,10 @@
|
||||
|
||||
## Storage and media
|
||||
|
||||
- Storage abstraction, backend configuration, logical buckets, object lifecycle, deduplication, access rules, and garbage collection: `mem:backend/storage`.
|
||||
- Storage has a fixed valid bucket set. Backends are `:fs` and `:s3`; default backend comes from deprecated `assets-storage-backend` only when present, otherwise `objects-storage-backend`, defaulting to `:fs`.
|
||||
- `put-object!` creates the DB `storage_object` row before writing backend content. Backend writes happen only for newly created rows, so deduplication can skip object writes.
|
||||
- Deduplication only applies when requested, when the content can provide a hash, and when bucket metadata is present. Reads exclude soft-deleted storage rows.
|
||||
- `sto/resolve` can reuse the current DB connection via `::db/reuse-conn true`; preserve this in transaction-sensitive code.
|
||||
- SVG validation strips DOCTYPE and uses secure SAX parsing. Basic SVG info falls back to 100x100 dimensions when width/height/viewBox are missing.
|
||||
- Raster metadata is shell-derived with ImageMagick `identify`, verifies detected MIME against the supplied MIME, and swaps dimensions for EXIF orientations 6/8.
|
||||
- Remote image download requires 2xx status, `content-length`, a known MIME, and size under the configured maximum before writing the temp file; mismatched byte count is an internal error.
|
||||
@@ -25,4 +28,4 @@
|
||||
- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data.
|
||||
- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob.
|
||||
- Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written.
|
||||
- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders.
|
||||
- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders.
|
||||
@@ -1,119 +0,0 @@
|
||||
# Backend Storage
|
||||
|
||||
## Abstraction
|
||||
|
||||
- `app.storage` stores binary objects.
|
||||
- Each object has a `storage_object` database row.
|
||||
- The row stores the UUID, size, backend, timestamps, and Transit metadata.
|
||||
- The backend stores the binary content.
|
||||
- Supported backends are `:fs` and `:s3`.
|
||||
- FS uses one root directory and a UUID-derived path.
|
||||
- S3 uses one configured bucket and an optional prefix.
|
||||
- A Penpot bucket is metadata. It is not an S3 bucket or a filesystem directory.
|
||||
- FS and S3 use the same UUID-derived object path. The bucket does not change the path.
|
||||
- `PENPOT_OBJECTS_STORAGE_*` configures the current object backend.
|
||||
- Deprecated asset-storage config keys remain supported for migration.
|
||||
- Database rows keep the backend name. Keep the legacy `:assets-fs` and `:assets-s3` aliases.
|
||||
|
||||
## Object Lifecycle
|
||||
|
||||
- `put-object!` creates the database row before it writes backend content.
|
||||
- Backend content is written only when the row is new.
|
||||
- A failed backend write can leave an unreferenced database row.
|
||||
- Callers often set `:touched-at` so garbage collection can remove such rows.
|
||||
- `get-object` excludes rows with `deleted_at`.
|
||||
- Existing object values can remain readable until physical deletion.
|
||||
- `:expired-at` blocks reads after the expiration time.
|
||||
- `del-object!` sets `deleted_at`. It does not remove backend content.
|
||||
- `storage-gc-deleted` removes the database row and backend content after the deletion delay.
|
||||
- `storage-gc-touched` finds references before it sets `deleted_at`.
|
||||
- `objects-gc` removes deleted domain rows and touches their storage object IDs.
|
||||
- Use `::db/reuse-conn true` with `sto/resolve` inside a database transaction.
|
||||
|
||||
## Connection Reuse Details
|
||||
|
||||
### `app.storage/resolve` patterns:
|
||||
|
||||
**1. Pool mode (default)** - `(sto/resolve cfg)`
|
||||
- Returns storage abstraction from config
|
||||
- Uses whatever database pool is available
|
||||
- **Safe to call outside transaction context**
|
||||
- Used in: `rpc/commands/media.clj:363`, `rpc/commands/auth.clj:327`, `rpc/commands/profile.clj:362`
|
||||
|
||||
**2. Connection reuse mode** - `(sto/resolve cfg ::db/reuse-conn true)`
|
||||
- Internally calls `db/get-connection cfg` to obtain connectable
|
||||
- Configures storage with the specific connection from config
|
||||
- **Must be paired with transaction that owns this connection**
|
||||
- Used in: `features/fdata.clj:100`, `rpc/commands/media.clj:425`, `rpc/commands/files_thumbnails.clj:307,319`, `binfile/v3.clj:722`
|
||||
|
||||
**3. Explicit configuration** - `(sto/configure storage conn)`
|
||||
- Sets `::db/conn` on storage map directly
|
||||
- Asserts `db/conn? connection` (storage.clj:349)
|
||||
- Used inside `db/tx-run!` blocks where `conn` is already available
|
||||
- Used in: `tasks/file_gc.clj:256`, `rpc/commands/files_thumbnails.clj:347,371`
|
||||
|
||||
### Key Warning (from function notes):
|
||||
|
||||
The improved note in `import-storage-objects` and `handle-persistence` warns:
|
||||
**Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions.
|
||||
|
||||
### Rule of Thumb for `sto/put-object!`:
|
||||
|
||||
Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `impl/put-object`) and does not directly use `::db/conn` or `::db/pool`, **all usage of `put-object!` will never run inside a common transaction** (if configured at all). The storage backend operations are independent of the database transaction boundary.
|
||||
|
||||
## Deduplication
|
||||
|
||||
- Deduplication requires `::sto/deduplicate?`, a content hash, and bucket metadata.
|
||||
- The lookup matches hash, bucket, backend, and `deleted_at IS NULL`.
|
||||
- The lookup only considers rows with `status='valid'`; pending rows are invisible.
|
||||
- A hit whose blob is missing is repaired in place: the same row/id is kept,
|
||||
and `put-object!` rewrites the blob under that id. This heals all existing
|
||||
references to the object. If the rewrite fails, the row is left live and
|
||||
valid for a later retry.
|
||||
- The lookup does not include file ID, profile ID, team ID, or organization ID.
|
||||
- Objects can therefore share content across users and files within one bucket.
|
||||
- Deleted objects are not reused.
|
||||
- `tempfile` objects never use deduplication, even when the caller requests it.
|
||||
- Use `sto/wrap-with-hash` when the caller already calculated the content hash.
|
||||
|
||||
## Bucket Rules
|
||||
|
||||
| Bucket | Content and references | Dedup | Direct `/assets/by-id` access | Cleanup |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `file-media-object` | Original file images and generated media thumbnails. References: `file_media_object.media_id` and `thumbnail_id`. | Yes | Public | Reference scan. |
|
||||
| `team-font-variant` | Font variants in `team_font_variant`. References: `woff1_file_id`, `woff2_file_id`, `otf_file_id`, and `ttf_file_id`. | Yes | Public | Reference scan. |
|
||||
| `file-object-thumbnail` | Frame and component thumbnails in `file_tagged_object_thumbnail.media_id`. | Yes | Public | Reference scan. |
|
||||
| `file-thumbnail` | File grid thumbnails in `file_thumbnail.media_id`. | Yes | Authentication required | Reference scan. |
|
||||
| `profile` | User and team profile photos. References: `profile.photo_id` and `team.photo_id`. | Yes | Authentication required | Reference scan. |
|
||||
| `organization` | Organization logos uploaded by the Nitrate management API. | Yes | Public | No reference scan. A touched object is deleted. |
|
||||
| `tempfile` | Export files, chunked-upload chunks, and temporary font downloads. | No | Authentication required | No reference scan. A touched object uses a two-hour deletion delay. |
|
||||
| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference metadata has `storage-ref-id`, `file-id`, and the `file_data` row ID. | Yes | Authentication required | Reference scan. |
|
||||
| `file-data-fragment` | Compatibility value for file-data fragments. The current backend has no dedicated producer for this bucket. | No current write semantics | Public | No touched-object collector case. |
|
||||
| `file-change` | Compatibility value for file changes. Current snapshots store data in `file_data`, not this bucket. | No current write semantics | Authentication required | No touched-object collector case. |
|
||||
|
||||
- The valid bucket set lives in `app.storage/valid-buckets`.
|
||||
- `file-media-object` is the default bucket for old rows without bucket metadata.
|
||||
- Do not assign a new bucket without adding its access and cleanup behavior.
|
||||
- The touched-object collector raises an internal error for an unknown bucket.
|
||||
- It supports `file-media-object`, `team-font-variant`, `file-object-thumbnail`, `file-thumbnail`, `profile`, `file-data`, `tempfile`, and `organization`.
|
||||
- It does not support `file-data-fragment` or `file-change`.
|
||||
|
||||
## Access Rules
|
||||
|
||||
- `app.http.assets` decides direct object authentication from the bucket.
|
||||
- Public buckets are `file-media-object`, `file-object-thumbnail`, `team-font-variant`, `file-data-fragment`, and `organization`.
|
||||
- Other valid buckets require a session or access-token profile ID.
|
||||
- File-media routes also require file read permission.
|
||||
- Non-public direct responses set `content-disposition: attachment`.
|
||||
- FS responses use `x-accel-redirect` for the configured asset path.
|
||||
- S3 responses use a presigned URL and an HTTP redirect.
|
||||
|
||||
## File Data
|
||||
|
||||
- `file-data-backend` accepts `legacy-db`, `db`, or `storage`.
|
||||
- `legacy-db` stores main data in `file.data` and snapshots in `file_change.data`.
|
||||
- `db` stores encoded data in `file_data.data`.
|
||||
- `storage` stores encoded data in storage subsystem with `file-data` bucket and keeps `data` nil in `file_data` table.
|
||||
- The `file_data.metadata.storage-ref-id` value points to the storage object.
|
||||
- `fdata/upsert!` touches a storage object from incoming metadata before it stores the new row.
|
||||
- File snapshots use `file_data` for snapshot data and `file_change` for snapshot metadata.
|
||||
@@ -1,11 +0,0 @@
|
||||
# Backend Testing
|
||||
|
||||
JVM `clojure.test` (kaocha runner) under `backend/test/backend_tests/`.
|
||||
|
||||
- READ `mem:testing` FIRST — it defines the execution discipline (no piping, tee to file, preferred commands) that applies to all JVM test runs.
|
||||
- All CLI commands must be executed from the `backend/` subdirectory.
|
||||
- Tests are invoked directly via `clojure -M:dev:test` (kaocha) — there is no pnpm wrapper. Kaocha auto-discovers test namespaces, so no runner registration is needed.
|
||||
- Coverage: if code is added or modified in `src/`, corresponding tests in `test/backend_tests/` must be added or updated.
|
||||
- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var.
|
||||
- Regression run: `clojure -M:dev:test` to ensure no regressions in related functional areas.
|
||||
- If you need to filter output, tee to a temp file first: `clojure -M:dev:test 2>&1 | tee /tmp/penpot-test-output.txt`.
|
||||
@@ -5,10 +5,9 @@
|
||||
## Layout and commands
|
||||
|
||||
- Source: `exporter/src/`; config: `deps.edn`, `shadow-cljs.edn`, `package.json`; runtime helpers/assets: `vendor/`, `scripts/`.
|
||||
- From `exporter/`: setup `./scripts/setup`; watch `pnpm run watch` or `pnpm run watch:app`; production build `pnpm run build`; test bundle `pnpm run build:test`; tests `pnpm run test` or `pnpm run test:quiet`; lint `pnpm run lint:clj`; format check/fix `pnpm run check-fmt:clj` / `pnpm run fmt:clj`.
|
||||
- From `exporter/`: setup `./scripts/setup`; watch `pnpm run watch` or `pnpm run watch:app`; production build `pnpm run build`; lint `pnpm run lint`; format check/fix `pnpm run check-fmt` / `pnpm run fmt`.
|
||||
- Because exporter consumes `common/`, shared file/shape/model changes may need exporter verification even when the immediate change is not under `exporter/`.
|
||||
- Cross-cutting testing principles and anti-patterns: `mem:testing`.
|
||||
- Exporter test conventions and CI: `mem:exporter/testing`.
|
||||
|
||||
## HTTP and browser pool
|
||||
|
||||
@@ -32,4 +31,4 @@
|
||||
- WebP is produced by taking a PNG screenshot and converting it with ImageMagick.
|
||||
- SVG export rasterizes text foreignObjects to PNG, converts through PPM/color masks/potrace, and reassembles SVG paths. It also replaces non-breaking spaces for SVG compatibility and drops empty defs/paths.
|
||||
- PDF export injects `@page` sizing through raw browser `evaluate` JavaScript; that code cannot rely on CLJS runtime helpers.
|
||||
- Temporary resources schedule local deletion, then uploads POST to `/api/management/methods/upload-tempfile` with `X-Shared-Key: exporter <management-key>` and Bearer auth.
|
||||
- Temporary resources schedule local deletion, then uploads POST to `/api/management/methods/upload-tempfile` with `X-Shared-Key: exporter <management-key>` and Bearer auth.
|
||||
@@ -1,16 +0,0 @@
|
||||
# Exporter Testing
|
||||
|
||||
- READ `mem:testing` first.
|
||||
- Tests use `cljs.test` and live under `exporter/test/exporter_tests/`.
|
||||
- Register every test namespace in `exporter-tests.runner`.
|
||||
- From `exporter/`: `pnpm run build:test` builds the Node test bundle without running tests.
|
||||
- From `exporter/`: `pnpm run test` builds and runs tests with full output.
|
||||
- From `exporter/`: `pnpm run test:quiet` builds and runs tests with reduced build output.
|
||||
- After `build:test`, reuse the compiled bundle with `node target/tests/test.js`.
|
||||
- For iterative focused runs, build once and reuse the compiled bundle.
|
||||
- Focus a test namespace with `node target/tests/test.js --focus exporter-tests.renderer-svg-test`.
|
||||
- Focus a test var with `node target/tests/test.js --focus exporter-tests.renderer-svg-test/creates-the-correct-gradient-element`.
|
||||
- Set app log level by appending `--log-level warn` (or `trace|debug|info|warn|error`).
|
||||
- `test:quiet` accepts forwarded options but rebuilds the bundle; prefer the direct runner after `build:test` for focused runs.
|
||||
- From `exporter/`: `pnpm run check-fmt:clj` checks ClojureScript formatting.
|
||||
- From `exporter/`: `pnpm run lint:clj` runs ClojureScript linting.
|
||||
@@ -6,7 +6,7 @@ Backend (`app.config`, `PENPOT_*` env vars) is parameterized; deployments choose
|
||||
|
||||
- **PostgreSQL**: durable store. Profiles, teams, files, sessions, audit, `storage_object` metadata, the `task` queue, `scheduled_task` cron registry, migrations. File-data also lives here when the file-data backend is `legacy-db`/`db`. One shared DB across all backends.
|
||||
- **Redis (Valkey-compatible)**: per-backend message bus and cache. Concrete uses: msgbus Pub/Sub for collaborative-editing broadcasts and team/profile-org notifications fired by RPC handlers (`app.rpc.notifications`, `files_update`, `teams`, `websocket`); file-summary cache gated by `enable-redis-cache`; rate-limit counters; and the dispatcher→runner work hand-off list `penpot.worker.queue:<tenant>:<queue>`. `PENPOT_REDIS_URI`.
|
||||
- **Object storage**: backends `:s3` and `:fs`. S3 in prod; devenv uses MinIO. Holds uploaded media, file-data when the file-data backend is `storage`, exports. Backend-side details (resolve, dedup, bucket set, object lifecycle, and file-data backends): `mem:backend/storage`.
|
||||
- **Object storage**: backends `:s3` and `:fs`. S3 in prod; devenv uses MinIO. Holds uploaded media, file-data when the file-data backend is `storage`, exports. Backend-side details (resolve, dedup, bucket set, file-data backends): `mem:backend/http-storage-filedata-subtleties`.
|
||||
- **SMTP mailer**: invitations, password resets, email verification (sent via the `:sendmail` worker task).
|
||||
- **LDAP** (optional auth provider): helpers in `app.auth.*`, gated by `enable-login-with-ldap`.
|
||||
|
||||
@@ -30,4 +30,4 @@ Penpot in production lives with both: horizontal-scale deployments accept "exact
|
||||
## See also
|
||||
|
||||
- Devenv composition and the ws0-only worker placement: `mem:devenv/core`.
|
||||
- Storage backend resolution, dedup, bucket behavior, object lifecycle, and file-data lifecycle: `mem:backend/storage`.
|
||||
- Storage backend resolution, dedup, file-data lifecycle: `mem:backend/http-storage-filedata-subtleties`.
|
||||
@@ -17,18 +17,9 @@
|
||||
|
||||
## Tile/render behavior
|
||||
|
||||
- Raster `Fill::Image`: skip `save_layer` unless the shape has an image filter; plain
|
||||
Rect/Frame (no corners) also skip the container clip (`draw_image_fill` in fills.rs).
|
||||
- Zoom settle: visible tiles present via `FrameType::ViewportReady` before interest-ring
|
||||
work; crop-cache rebuild is deferred to the later `Full` so the soft→sharp snap is
|
||||
compose+present only.
|
||||
- Interactive transforms are distinct from viewport fast mode. `set_modifiers_start` enables fast mode and interactive transform; interactive transform still flushes each animation frame.
|
||||
- During interactive transform, modifier tile invalidation is deferred to `render()` once per rAF. Outside interactive transform, `set_modifiers` rebuilds modifier tiles immediately.
|
||||
- `set_modifiers_end` disables fast/interactive state and cancels pending async render; the caller must request the final full-quality render.
|
||||
- Plain viewport fast mode (`options.is_viewport_interaction()`) renders from cache and does not flush target output inside `process_animation_frame`; interactive transforms do flush.
|
||||
- Zoom settle wipes the tile texture cache in `set_view_end`. Mid-zoom overlays
|
||||
key tiles by scale; shape edits must `invalidate_cached_tiles_intersecting`
|
||||
the old∪new extrect so those overlays do not keep pre-edit pixels.
|
||||
- Pending tile priority is intentionally reversed by pop order; check the queue construction before changing tile scheduling.
|
||||
- Frames with a fill may use `render_frame_container_drop_shadow` (direct rrect +
|
||||
blur saveLayer on `DropShadows`) when `uses_direct_container_drop_shadow` is true.
|
||||
- Zoom changes rebuild the tile index while preserving cached tile textures. Avoid replacing that path with shallow rebuilds if blur/shadow cache preservation matters.
|
||||
- Pending tile priority is intentionally reversed by pop order; check the queue construction before changing tile scheduling.
|
||||
@@ -9,7 +9,6 @@ repository via GraphQL and REST APIs through the authenticated `gh` CLI.
|
||||
- Finding issues with no milestone.
|
||||
- Fetching PR details by number or by milestone.
|
||||
- Comparing milestone issues against CHANGES.md to find missing entries.
|
||||
- Listing or inspecting GitHub Security Advisories (GHSA).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -73,30 +72,6 @@ python3 scripts/gh.py prs --milestone "2.16.0" --state all
|
||||
|
||||
**Output**: JSON array to stdout; progress to stderr.
|
||||
|
||||
### `advisories`
|
||||
|
||||
List or inspect GitHub Security Advisories for the repository.
|
||||
|
||||
```bash
|
||||
# List all advisories (summary view)
|
||||
python3 scripts/gh.py advisories
|
||||
|
||||
# Filter by severity
|
||||
python3 scripts/gh.py advisories --severity critical
|
||||
|
||||
# Filter by state
|
||||
python3 scripts/gh.py advisories --state triage
|
||||
|
||||
# Get full detail for a single advisory
|
||||
python3 scripts/gh.py advisories GHSA-xvj6-fh9w-gjw7
|
||||
```
|
||||
|
||||
**Summary output fields**: ghsa_id, cve_id, severity, cvss_score, state, summary, cwes, published_at, closed_at, url.
|
||||
|
||||
**Detail output** (single advisory) adds: description, vulnerabilities (package, version ranges), credits, timestamps.
|
||||
|
||||
**Output**: JSON to stdout; progress to stderr.
|
||||
|
||||
## Key principles
|
||||
|
||||
- All output is JSON — pipe into `jq` or other tools for further processing.
|
||||
|
||||
@@ -13,7 +13,7 @@ and helpers, consult:
|
||||
builders, production-path change helpers
|
||||
- `mem:frontend/testing` — CLJS unit tests, Playwright E2E integration tests,
|
||||
live browser verification via nREPL
|
||||
- `mem:backend/testing` — JVM `clojure.test` under `backend/test/`
|
||||
- Backend — JVM `clojure.test` under `backend/test/`; see `mem:backend/core`
|
||||
|
||||
## When to Use
|
||||
|
||||
|
||||
@@ -34,19 +34,6 @@ Skipping this step is the #1 cause of incorrect or incomplete work.
|
||||
|
||||
---
|
||||
|
||||
## Auto-triggers
|
||||
|
||||
- **Security advisory URL pasted** — When the user pastes a URL matching
|
||||
`github.com/penpot/penpot/security/advisories/GHSA-*`, extract the GHSA ID
|
||||
from the URL and run `python3 scripts/gh.py advisories <GHSA-ID>` to fetch
|
||||
full advisory details before proceeding.
|
||||
|
||||
## Writing Rules
|
||||
|
||||
Use the `ste` skill when the user explicitly requests STE, `/ste`, or ASD-STE100.
|
||||
|
||||
---
|
||||
|
||||
# Memory system
|
||||
|
||||
Memories are the **primary project guidance** — not docs or readme files.
|
||||
@@ -126,5 +113,4 @@ precision while maintaining a strong focus on maintainability and performance.
|
||||
- `scripts/check-commit` — Validate commit messages against Penpot's commit guidelines.
|
||||
- `scripts/check-fmt-clj` — Check Clojure formatting without modifying files.
|
||||
- `scripts/ci` — CI orchestration script for running lint, tests, and format checks across modules. See `scripts/ci --help`.
|
||||
- `scripts/gh.py` — Multi-purpose GitHub CLI helper. Subcommands: `issues` (list issues in a milestone), `prs` (fetch PR details), `advisories` (list/inspect security advisories). See `python3 scripts/gh.py --help`.
|
||||
|
||||
+3
-25
@@ -23,39 +23,17 @@
|
||||
- Refactor wasm rulers and UI state [#10116](https://github.com/penpot/penpot/issues/10116) (PR: [#10461](https://github.com/penpot/penpot/pull/10461))
|
||||
- Improve team invitations modal in the dashboard [#10484](https://github.com/penpot/penpot/issues/10484) (PR: [#10459](https://github.com/penpot/penpot/pull/10459))
|
||||
|
||||
## 2.17.2
|
||||
|
||||
## 2.17.1 (Unreleased)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix linear gradients in SVG text exports being emitted as radial gradients [#5972](https://github.com/penpot/penpot/issues/5972) (PR: [#11272](https://github.com/penpot/penpot/pull/11272))
|
||||
- Fix typography token becoming detached when editing text content [#11362](https://github.com/penpot/penpot/issues/11362) (PR: [#11366](https://github.com/penpot/penpot/pull/11366))
|
||||
- Fix command injection in SVG exporter via legacy fill-color (https://github.com/penpot/penpot/security/advisories/GHSA-4f36-m4hj-cv86)
|
||||
|
||||
## 2.17.1
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix overrides lost after switching component variant [#10588](https://github.com/penpot/penpot/issues/10588) (PR: [#10619](https://github.com/penpot/penpot/pull/10619))
|
||||
- Fix malformed get-font-variants request when team-id is missing from dashboard URL [#10644](https://github.com/penpot/penpot/issues/10644) (PR: [#10645](https://github.com/penpot/penpot/pull/10645))
|
||||
- Fix malformed get-profiles-for-file-comments request when file-id is missing from workspace URL [#10652](https://github.com/penpot/penpot/issues/10652) (PR: [#10655](https://github.com/penpot/penpot/pull/10655))
|
||||
- Fix internal error when dragging inner layout with Boolean operations [#10647](https://github.com/penpot/penpot/issues/10647) (PR: [#10778](https://github.com/penpot/penpot/pull/10778))
|
||||
- Fix frontend throwing raw TypeError on undefined .getData receivers across import, paste, drag, and text editor paths [#10709](https://github.com/penpot/penpot/issues/10709) (PR: [#10718](https://github.com/penpot/penpot/pull/10718))
|
||||
- Fix workspace crash with 'can't access dead object' in Firefox when navigating between pages [#10719](https://github.com/penpot/penpot/issues/10719) (PR: [#10721](https://github.com/penpot/penpot/pull/10721))
|
||||
- Fix workspace crash when holding an arrow key on a selection due to excessive re-renders [#10726](https://github.com/penpot/penpot/issues/10726) (PR: [#10736](https://github.com/penpot/penpot/pull/10736))
|
||||
- Fix dashboard sidebar throwing removeChild NotFoundError during rapid keyboard navigation [#10714](https://github.com/penpot/penpot/issues/10714) (PR: [#10715](https://github.com/penpot/penpot/pull/10715))
|
||||
- Fix asset download failing with S3 auth conflict when using access token [#10776](https://github.com/penpot/penpot/issues/10776) (PR: [#10777](https://github.com/penpot/penpot/pull/10777))
|
||||
- Fix import worker crashing when importing non-Penpot zip files [#10781](https://github.com/penpot/penpot/issues/10781) (PR: [#10782](https://github.com/penpot/penpot/pull/10782))
|
||||
- Fix internal error when dragging inner layout with Boolean operations [#10647](https://github.com/penpot/penpot/issues/10647) (PR: [#10778](https://github.com/penpot/penpot/pull/10778))
|
||||
- Fix viewer crash with WASM panic when opening URL with page-id [#10800](https://github.com/penpot/penpot/issues/10800) (PR: [#10805](https://github.com/penpot/penpot/pull/10805))
|
||||
- Fix backend returning 500 when JSON request body has unrecognized escape sequence [#10804](https://github.com/penpot/penpot/issues/10804) (PR: [#10808](https://github.com/penpot/penpot/pull/10808))
|
||||
- Fix color picker eyedropper crashing when viewport is unmounted during pointer move [#10811](https://github.com/penpot/penpot/issues/10811) (PR: [#10812](https://github.com/penpot/penpot/pull/10812))
|
||||
- Fix flex layout crash when dragging shapes with missing bounds [#10843](https://github.com/penpot/penpot/issues/10843) (PR: [#10845](https://github.com/penpot/penpot/pull/10845))
|
||||
- Fix export failing when shape has blank layer name [#10849](https://github.com/penpot/penpot/issues/10849) (PR: [#10852](https://github.com/penpot/penpot/pull/10852))
|
||||
- Fix area selection (marquee) being aborted by select-shapes interrupt [#10872](https://github.com/penpot/penpot/issues/10872) (PR: [#10870](https://github.com/penpot/penpot/pull/10870))
|
||||
- Fix gradient editor sending invalid stop offset when clicking outside gradient line [#10879](https://github.com/penpot/penpot/issues/10879) (PR: [#10881](https://github.com/penpot/penpot/pull/10881))
|
||||
- Fix audit event validation failing when error reports contain string profile-id and missing token context [#10897](https://github.com/penpot/penpot/issues/10897) (PR: [#10898](https://github.com/penpot/penpot/pull/10898))
|
||||
- Fix MCP tool call timeout being too low for some operations [#10953](https://github.com/penpot/penpot/issues/10953) (PR: [#10967](https://github.com/penpot/penpot/pull/10967))
|
||||
- Fix MCP requests running into timeouts after leaving a file in Penpot [#10958](https://github.com/penpot/penpot/issues/10958) (PR: [#10967](https://github.com/penpot/penpot/pull/10967))
|
||||
- Fix duplicate WebSocket MCP connection attempts deregistering the original connection's routing entries [#10961](https://github.com/penpot/penpot/issues/10961) (PR: [#10967](https://github.com/penpot/penpot/pull/10967))
|
||||
|
||||
## 2.17.0
|
||||
|
||||
|
||||
@@ -160,6 +160,6 @@ This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
```
|
||||
Penpot is a Kaleidos’ [open source project](https://kaleidos.net/)
|
||||
+1
-2
@@ -48,7 +48,6 @@
|
||||
|
||||
buddy/buddy-hashers {:mvn/version "2.0.167"}
|
||||
buddy/buddy-sign {:mvn/version "3.6.1-359"}
|
||||
org.passay/passay {:mvn/version "1.6.6"}
|
||||
|
||||
com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.4"}
|
||||
|
||||
@@ -78,7 +77,7 @@
|
||||
org.clojure/data.csv {:mvn/version "1.1.1"}
|
||||
com.clojure-goes-fast/clj-async-profiler {:mvn/version "2.0.0-beta1"}
|
||||
mockery/mockery {:mvn/version "0.1.4"}}
|
||||
:extra-paths ["test" "dev"]}
|
||||
:extra-paths ["test" "test/resources" "dev"]}
|
||||
|
||||
:build
|
||||
{:extra-deps
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
;; This is an example on how it can be executed:
|
||||
;; clojure -Scp $(cat classpath) -M dev/script-fix-sobjects.clj
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns user
|
||||
(:require
|
||||
|
||||
@@ -190,20 +190,6 @@ Debug Main Page
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Validate file:</legend>
|
||||
<desc>Given an FILE-ID, check the referential integrity.</desc>
|
||||
<form method="get" action="/dbg/actions/file-validate">
|
||||
<div class="row">
|
||||
<input type="text" style="width:300px" name="file-id" placeholder="file-id" />
|
||||
</div>
|
||||
<div class="row">
|
||||
<input type="submit" name="validate" value="Validate" />
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
</section>
|
||||
<section class="widget">
|
||||
<fieldset>
|
||||
@@ -236,7 +222,6 @@ Debug Main Page
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Import binfile:</legend>
|
||||
<desc>Import penpot file in binary format.</desc>
|
||||
@@ -251,34 +236,6 @@ Debug Main Page
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Repair file:</legend>
|
||||
<desc>Given an FILE-ID, repair the referential integrity errors.
|
||||
<br/>
|
||||
<br/>
|
||||
<b>WARNING: the reparation is not guaranteed and may cause loss of data!</b>
|
||||
<br/>
|
||||
<br/>
|
||||
You may need to give several repair rounds until all errors are cleared.
|
||||
</desc>
|
||||
<form method="get" action="/dbg/actions/file-repair">
|
||||
<div class="row">
|
||||
<input type="text" style="width:300px" name="file-id" placeholder="file-id" />
|
||||
</div>
|
||||
<div class="row">
|
||||
<label for="check-snapshot">Skip snapshot</label>
|
||||
<input id="check-snapshot" type="checkbox" name="skip-snapshot" />
|
||||
<br />
|
||||
<small>
|
||||
A snapshot is made just before the validation, unless skipped.
|
||||
</small>
|
||||
</div>
|
||||
<div class="row">
|
||||
<input type="submit" name="repair" value="Repair" />
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
@@ -39,16 +39,4 @@
|
||||
{:permits 3}
|
||||
|
||||
:create-file-snapshot/by-profile
|
||||
{:permits 1 :queue 2 :timeout 60000}
|
||||
|
||||
:send-user-feedback/global
|
||||
{:permits 4}
|
||||
|
||||
:send-user-feedback/by-profile
|
||||
{:permits 1 :queue 3}
|
||||
|
||||
:import-binfile/global
|
||||
{:permits 4}
|
||||
|
||||
:import-binfile/by-profile
|
||||
{:permits 1 :queue 2}}
|
||||
{:permits 1 :queue 2 :timeout 60000}}
|
||||
@@ -1,308 +1,11 @@
|
||||
;; Example rlimit.edn file
|
||||
^{:refresh "30s"}
|
||||
{:default
|
||||
[[:default :window "200000/h"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Auth & Identity — public, unauthenticated
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/login-with-password}
|
||||
[[:auth-password :bucket "100/50/1m"]]
|
||||
;; #{:main/get-teams}
|
||||
;; [[:burst :bucket "5/5/5s"]]
|
||||
|
||||
#{:main/login-with-ldap}
|
||||
[[:auth-ldap :bucket "20/10/5m"]]
|
||||
|
||||
#{:main/register-profile}
|
||||
[[:auth-register :bucket "20/10/15m"]]
|
||||
|
||||
#{:main/request-profile-recovery
|
||||
:main/prepare-register-profile}
|
||||
[[:auth-recovery :bucket "100/50/5m"]]
|
||||
|
||||
#{:main/recover-profile
|
||||
:main/verify-token}
|
||||
[[:auth-token :bucket "100/50/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; SSRF vectors — URL fetch endpoints
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-file-media-object-from-url}
|
||||
[[:url-fetch :bucket "100/50/5m"]]
|
||||
|
||||
#{:main/create-webhook
|
||||
:main/update-webhook}
|
||||
[[:webhook-validation :bucket "20/10/5m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Search — full sequential scan risk
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/search-files}
|
||||
[[:search :bucket "60/30/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Feedback & Invitations — email-sending
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/send-user-feedback
|
||||
:main/create-team-invitations}
|
||||
[[:email-send :bucket "30/15/5m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Media & File heavy ops
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/upload-file-media-object}
|
||||
[[:image-upload :bucket "200/100/1m"]]
|
||||
|
||||
#{:main/create-file-object-thumbnail
|
||||
:main/delete-file-object-thumbnails
|
||||
:main/get-file-object-thumbnails}
|
||||
[[:thumbnail-ops :bucket "5000/3000/1m"]]
|
||||
|
||||
#{:main/get-file-data-for-thumbnail
|
||||
:main/create-file-thumbnail}
|
||||
[[:thumbnail-data :bucket "100/50/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; UI navigation reads — high frequency
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/get-teams}
|
||||
[[:get-teams :bucket "5000/2500/30s"]]
|
||||
|
||||
#{:main/get-team-members}
|
||||
[[:get-team-members :bucket "4000/2000/30s"]]
|
||||
|
||||
#{:main/get-profile}
|
||||
[[:get-profile :bucket "500/250/30s"]]
|
||||
|
||||
#{:main/get-font-variants}
|
||||
[[:get-font-variants :bucket "250/125/30s"]]
|
||||
|
||||
#{:main/get-comment-threads}
|
||||
[[:get-comment-threads :bucket "500/250/30s"]]
|
||||
|
||||
#{:main/get-profiles-for-file-comments}
|
||||
[[:get-profiles-for-file-comments :bucket "300/150/30s"]]
|
||||
|
||||
#{:main/get-file-libraries}
|
||||
[[:get-file-libraries :bucket "200/100/30s"]]
|
||||
|
||||
#{:main/get-projects}
|
||||
[[:get-projects :bucket "120/60/30s"]]
|
||||
|
||||
#{:main/get-team-recent-files
|
||||
:main/get-unread-comment-threads}
|
||||
[[:get-team-recent :bucket "120/60/30s"]]
|
||||
|
||||
#{:main/get-page}
|
||||
[[:get-page :bucket "150/75/30s"]]
|
||||
|
||||
#{:main/get-access-tokens
|
||||
:main/get-subscription-usage}
|
||||
[[:get-access-tokens :bucket "150/75/30s"]]
|
||||
|
||||
#{:main/get-enabled-flags}
|
||||
[[:get-enabled-flags :bucket "250/125/30s"]]
|
||||
|
||||
#{:main/get-builtin-templates}
|
||||
[[:get-builtin-templates :bucket "200/100/30s"]]
|
||||
|
||||
#{:main/get-project
|
||||
:main/get-project-files}
|
||||
[[:get-project-info :bucket "80/40/30s"]]
|
||||
|
||||
#{:main/get-file}
|
||||
[[:get-file :bucket "180/90/1m"]]
|
||||
|
||||
#{:main/get-team-shared-files
|
||||
:main/get-team-info
|
||||
:main/get-team-users
|
||||
:main/get-team-invitations
|
||||
:main/get-team-deleted-files
|
||||
:main/get-sso-provider}
|
||||
[[:get-team-info :bucket "60/30/30s"]]
|
||||
|
||||
#{:main/get-comments
|
||||
:main/get-file-snapshots
|
||||
:main/get-library-usage
|
||||
:main/has-file-libraries}
|
||||
[[:get-misc-list :bucket "300/150/30s"]]
|
||||
|
||||
#{:main/get-comment-thread
|
||||
:main/get-library-file-references}
|
||||
[[:get-misc-single :bucket "60/30/30s"]]
|
||||
|
||||
#{:main/get-file-info
|
||||
:main/get-view-only-bundle
|
||||
:main/get-all-projects
|
||||
:main/get-owned-teams
|
||||
:main/get-team-stats
|
||||
:main/get-file-summary
|
||||
:main/get-file-stats
|
||||
:main/get-file-fragment}
|
||||
[[:get-light :bucket "60/30/30s"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; File mutations — editing active
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/update-file}
|
||||
[[:update-file :bucket "1000/500/1m"]]
|
||||
|
||||
#{:main/create-file
|
||||
:main/rename-file
|
||||
:main/duplicate-file
|
||||
:main/move-files}
|
||||
[[:file-create :bucket "60/30/1m"]]
|
||||
|
||||
#{:main/delete-file}
|
||||
[[:file-delete :bucket "80/40/1m"]]
|
||||
|
||||
#{:main/set-file-shared
|
||||
:main/update-file-library-sync-status
|
||||
:main/ignore-file-library-sync-status
|
||||
:main/link-file-to-library
|
||||
:main/unlink-file-from-library
|
||||
:main/create-file-snapshot
|
||||
:main/restore-file-snapshot
|
||||
:main/update-file-snapshot
|
||||
:main/delete-file-snapshot
|
||||
:main/lock-file-snapshot
|
||||
:main/unlock-file-snapshot}
|
||||
[[:file-mutations :bucket "80/40/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Project mutations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-project}
|
||||
[[:project-create :bucket "100/50/1m"]]
|
||||
|
||||
#{:main/delete-project
|
||||
:main/rename-project
|
||||
:main/duplicate-project
|
||||
:main/move-project
|
||||
:main/update-project-pin}
|
||||
[[:project-mutations :bucket "40/20/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Team mutations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-team
|
||||
:main/update-team
|
||||
:main/delete-team
|
||||
:main/update-team-photo
|
||||
:main/update-team-member-role
|
||||
:main/delete-team-member
|
||||
:main/leave-team
|
||||
:main/create-team-with-invitations
|
||||
:main/create-team-access-request
|
||||
:main/permanently-delete-team-files
|
||||
:main/restore-deleted-team-files}
|
||||
[[:team-mutations :bucket "60/30/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Comment operations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-comment-thread
|
||||
:main/create-comment
|
||||
:main/update-comment
|
||||
:main/delete-comment
|
||||
:main/mark-all-threads-as-read}
|
||||
[[:comment-basic :bucket "30/15/1m"]]
|
||||
|
||||
#{:main/update-comment-thread
|
||||
:main/update-comment-thread-status
|
||||
:main/update-comment-thread-position
|
||||
:main/update-comment-thread-frame
|
||||
:main/delete-comment-thread}
|
||||
[[:comment-thread :bucket "80/40/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Profile operations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/update-profile
|
||||
:main/update-profile-props
|
||||
:main/update-profile-photo
|
||||
:main/update-profile-password
|
||||
:main/update-profile-notifications
|
||||
:main/delete-profile
|
||||
:main/delete-profile-photo
|
||||
:main/request-email-change}
|
||||
[[:profile-mutations :bucket "30/15/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Font operations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-font-variant
|
||||
:main/delete-font
|
||||
:main/delete-font-variant
|
||||
:main/update-font
|
||||
:main/download-font
|
||||
:main/download-font-family}
|
||||
[[:font-ops :bucket "100/50/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Access tokens
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-access-token
|
||||
:main/delete-access-token}
|
||||
[[:access-token :bucket "60/30/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Export / Import
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/export-binfile
|
||||
:main/import-binfile
|
||||
:main/clone-template}
|
||||
[[:export-import :bucket "80/40/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Upload sessions
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-upload-session
|
||||
:main/upload-chunk
|
||||
:main/assemble-file-media-object}
|
||||
[[:upload-session :bucket "100/50/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Webhooks
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/get-webhooks
|
||||
:main/delete-webhook}
|
||||
[[:webhook-read :bucket "20/10/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Share links
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/create-share-link
|
||||
:main/delete-share-link}
|
||||
[[:share-link :bucket "10/5/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Organization operations
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/add-team-to-organization
|
||||
:main/remove-team-from-org
|
||||
:main/all-org-members-in-team
|
||||
:main/all-team-members-in-orgs
|
||||
:main/get-owned-organizations-summary
|
||||
:main/get-leave-org-summary
|
||||
:main/leave-org
|
||||
:main/check-org-members
|
||||
:main/get-team-invitation-token
|
||||
:main/delete-team-invitation
|
||||
:main/check-team-external-invitations}
|
||||
[[:org-ops :bucket "20/10/1m"]]
|
||||
|
||||
;; ═══════════════════════════════════════════════
|
||||
;; Audit & stats
|
||||
;; ═══════════════════════════════════════════════
|
||||
#{:main/push-audit-events}
|
||||
[[:audit-events :bucket "1000/500/1m"]]
|
||||
|
||||
#{:main/logout
|
||||
:main/get-error-report
|
||||
:main/get-error-reports
|
||||
:main/get-current-mcp-token
|
||||
:main/get-nitrate-connectivity
|
||||
:main/check-nitrate-sso
|
||||
:main/redeem-nitrate-activation-code
|
||||
:main/create-demo-profile
|
||||
:main/get-subscription-warning}
|
||||
[[:misc-light :bucket "100/50/1m"]]}
|
||||
;; #{:main/get-profile}
|
||||
;; [[:burst :bucket "60/60/1m"]]
|
||||
}
|
||||
@@ -13,10 +13,6 @@ export PENPOT_MANAGEMENT_API_KEY=super-secret-management-api-key
|
||||
# PENPOT_DATABASE_*, PENPOT_REDIS_URI, PENPOT_OBJECTS_STORAGE_*, AWS_*) is owned by
|
||||
# docker/devenv/defaults.env and injected via the main service's env block.
|
||||
|
||||
if [ -f /home/selfsigned.crt ]; then
|
||||
export NODE_EXTRA_CA_CERTS=/home/selfsigned.crt;
|
||||
fi
|
||||
|
||||
# Background worker flag is per-instance. Defaults to enabled (ws0); ws1+
|
||||
# overlays set PENPOT_BACKEND_WORKER=false so scheduled and async tasks only
|
||||
# run on ws0, keeping notification Pub/Sub bound to a single Valkey. See
|
||||
@@ -48,7 +44,6 @@ export PENPOT_FLAGS="\
|
||||
enable-user-feedback \
|
||||
disable-secure-session-cookies \
|
||||
enable-smtp \
|
||||
enable-account-lockout \
|
||||
enable-prepl-server \
|
||||
enable-urepl-server \
|
||||
enable-nrepl-server \
|
||||
@@ -106,3 +101,5 @@ function setup_minio() {
|
||||
mc alias set penpot-s3/ "${PENPOT_OBJECTS_STORAGE_S3_ENDPOINT}" minioadmin minioadmin -q
|
||||
mc mb "penpot-s3/${PENPOT_OBJECTS_STORAGE_S3_BUCKET}" -p -q
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
# Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.auth
|
||||
(:require
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.auth.ldap
|
||||
(:require
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.auth.login-lockout
|
||||
"Brute-force protection: per-account failed login counter backed by Redis.
|
||||
Uses an atomic Lua script for the increment operation to prevent
|
||||
concurrent login bypass. Stores count:window_start in the value;
|
||||
time is passed from Clojure (ct/now) via ARGV, keeping it injectable
|
||||
for tests. When the counter reaches the configured threshold within
|
||||
the time window, the account is locked out until the window expires."
|
||||
(:require
|
||||
[app.common.generic-pool :as gpool]
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.config :as cf]
|
||||
[app.redis :as rds]
|
||||
[app.redis.script :as-alias rscript]
|
||||
[clojure.string :as str])
|
||||
(:import
|
||||
java.lang.AutoCloseable))
|
||||
|
||||
(def ^:private key-prefix "penpot.login-lockout.")
|
||||
|
||||
(def ^:private lockout-script
|
||||
{::rscript/name ::login-lockout
|
||||
::rscript/path "app/auth/login_lockout.lua"})
|
||||
|
||||
(defn- get-pool
|
||||
[cfg]
|
||||
(:app.redis/pool cfg))
|
||||
|
||||
(defn- with-conn
|
||||
[cfg f]
|
||||
(let [pool (get-pool cfg)
|
||||
conn (gpool/get pool)]
|
||||
(try
|
||||
(f @conn)
|
||||
(finally
|
||||
(.close ^AutoCloseable conn)))))
|
||||
|
||||
(defn- parse-value
|
||||
"Parse stored \"count:window_start\" string. Returns nil if missing
|
||||
or expired. Throws NumberFormatException for malformed values
|
||||
(caught by caller → fail-open)."
|
||||
[value window-ms now-ms]
|
||||
(when value
|
||||
(let [sep (str/index-of value ":")]
|
||||
(when sep
|
||||
(let [count (Long/parseLong (subs value 0 sep))
|
||||
start (Long/parseLong (subs value (inc sep)))]
|
||||
(when (> (+ start window-ms) now-ms)
|
||||
{:count count :window-start start}))))))
|
||||
|
||||
(defn record-failed-attempt!
|
||||
"Increment the failed-login counter for a profile-id (atomic via Lua).
|
||||
Returns nil when the flag is disabled or on Redis error (fail-open).
|
||||
Otherwise returns a map with :count (int), :ttl (int, seconds),
|
||||
and :locked? (boolean)."
|
||||
[cfg profile-id]
|
||||
(when (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(let [threshold (cf/get :login-lockout-max-attempts)
|
||||
window (cf/get :login-lockout-window)
|
||||
window-ms (if (integer? window) window (.toMillis window))
|
||||
_ (assert (>= threshold 1) "login-lockout-max-attempts must be >= 1")
|
||||
_ (assert (>= window-ms 60000) "login-lockout-window must be >= 60000ms (1 minute)")
|
||||
key (str key-prefix profile-id)
|
||||
now-ms (inst-ms (ct/now))
|
||||
result (with-conn cfg
|
||||
(fn [conn]
|
||||
(rds/eval conn
|
||||
(assoc lockout-script
|
||||
::rscript/keys [key]
|
||||
::rscript/vals [threshold window-ms now-ms]))))
|
||||
[count locked ttl] result]
|
||||
{:count count
|
||||
:locked? (= 1 locked)
|
||||
:ttl (max 0 ttl)})
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failing open on login lockout"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)
|
||||
nil))))
|
||||
|
||||
(defn clear-attempts!
|
||||
"Clear the failed-login counter (on successful login or password reset).
|
||||
No-op when the flag is disabled."
|
||||
[cfg profile-id]
|
||||
(when (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(with-conn cfg
|
||||
(fn [conn]
|
||||
(rds/del conn (str key-prefix profile-id))))
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failed to clear login lockout"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)))))
|
||||
|
||||
(defn locked?
|
||||
"Check whether the account is currently locked out. Does not increment
|
||||
the counter. Returns {:locked? false} when the flag is disabled or on
|
||||
Redis error (fail-open). When locked, includes :ttl (seconds remaining)."
|
||||
[cfg profile-id]
|
||||
(if (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(let [threshold (cf/get :login-lockout-max-attempts)
|
||||
window (cf/get :login-lockout-window)
|
||||
window-ms (if (integer? window) window (.toMillis window))
|
||||
_ (assert (>= threshold 1) "login-lockout-max-attempts must be >= 1")
|
||||
_ (assert (>= window-ms 60000) "login-lockout-window must be >= 60000ms (1 minute)")
|
||||
key (str key-prefix profile-id)
|
||||
now-ms (inst-ms (ct/now))
|
||||
result (with-conn cfg
|
||||
(fn [conn]
|
||||
(if-let [current (parse-value (rds/get conn key) window-ms now-ms)]
|
||||
(let [elapsed (- now-ms (:window-start current))
|
||||
ttl-ms (- window-ms elapsed)
|
||||
locked? (>= (:count current) threshold)]
|
||||
(cond-> {:locked? locked?}
|
||||
locked?
|
||||
(assoc :ttl (int (Math/ceil (/ ttl-ms 1000.0))))))
|
||||
{:locked? false})))]
|
||||
result)
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failing open on lockout check"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)
|
||||
{:locked? false}))
|
||||
{:locked? false}))
|
||||
@@ -1,46 +0,0 @@
|
||||
-- This Source Code Form is subject to the terms of the Mozilla Public
|
||||
-- License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
--
|
||||
-- Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
local key = KEYS[1]
|
||||
local threshold = tonumber(ARGV[1])
|
||||
local window_ms = tonumber(ARGV[2])
|
||||
local now_ms = tonumber(ARGV[3])
|
||||
local window_sec = math.ceil(window_ms / 1000)
|
||||
|
||||
local val = redis.call('GET', key)
|
||||
local count
|
||||
local window_start
|
||||
|
||||
if val then
|
||||
local colon = string.find(val, ':')
|
||||
if colon then
|
||||
count = tonumber(string.sub(val, 1, colon - 1))
|
||||
window_start = tonumber(string.sub(val, colon + 1))
|
||||
else
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
else
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
|
||||
if (now_ms - window_start) > window_ms then
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
|
||||
count = count + 1
|
||||
redis.call('SET', key, count .. ':' .. window_start, 'EX', window_sec)
|
||||
|
||||
local locked = 0
|
||||
local ttl = 0
|
||||
if count >= threshold then
|
||||
locked = 1
|
||||
ttl = math.ceil((window_ms - (now_ms - window_start)) / 1000)
|
||||
end
|
||||
|
||||
return {count, locked, ttl}
|
||||
+80
-250
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.auth.oidc
|
||||
"OIDC client implementation."
|
||||
@@ -42,52 +42,31 @@
|
||||
;; OIDC PROVIDER (GENERIC)
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- raise-invalid-sso-config
|
||||
"Raise a controlled validation error for OIDC provider configuration failures."
|
||||
[& {:keys [hint cause] :as params}]
|
||||
(throw (ex-info (or hint "invalid-sso-config")
|
||||
(-> params
|
||||
(dissoc :cause)
|
||||
(assoc :type :validation
|
||||
:code :invalid-sso-config))
|
||||
cause)))
|
||||
|
||||
(defn- discover-oidc-config
|
||||
[cfg {:keys [base-uri skip-ssrf-check?] :as provider}]
|
||||
(let [uri (u/join base-uri ".well-known/openid-configuration")]
|
||||
(try
|
||||
(let [rsp (http/req cfg {:method :get :uri (dm/str uri)} {:skip-ssrf-check? skip-ssrf-check?})]
|
||||
(if (= 200 (:status rsp))
|
||||
(let [data (-> rsp :body json/decode)
|
||||
token-uri (get data :token_endpoint)
|
||||
auth-uri (get data :authorization_endpoint)
|
||||
user-uri (get data :userinfo_endpoint)
|
||||
jwks-uri (get data :jwks_uri)
|
||||
logout-uri (get data :end_session_endpoint)]
|
||||
(let [uri (u/join base-uri ".well-known/openid-configuration")
|
||||
rsp (http/req cfg {:method :get :uri (dm/str uri)} {:skip-ssrf-check? skip-ssrf-check?})]
|
||||
|
||||
(-> provider
|
||||
(assoc :token-uri token-uri)
|
||||
(assoc :auth-uri auth-uri)
|
||||
(assoc :user-uri user-uri)
|
||||
(assoc :jwks-uri jwks-uri)
|
||||
(assoc :logout-uri logout-uri)))
|
||||
(if (= 200 (:status rsp))
|
||||
(let [data (-> rsp :body json/decode)
|
||||
token-uri (get data :token_endpoint)
|
||||
auth-uri (get data :authorization_endpoint)
|
||||
user-uri (get data :userinfo_endpoint)
|
||||
jwks-uri (get data :jwks_uri)
|
||||
logout-uri (get data :end_session_endpoint)]
|
||||
|
||||
(raise-invalid-sso-config
|
||||
:hint "unable to discover OIDC configuration"
|
||||
:discover-uri uri
|
||||
:response-status-code (:status rsp))))
|
||||
(catch Throwable cause
|
||||
;; Controlled raises above are ExceptionInfo and would otherwise be
|
||||
;; re-wrapped by this catch, dropping fields like :response-status-code.
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw cause)
|
||||
;; Wrap SSRF blocks, DNS failures, TLS errors, etc. — from the caller's
|
||||
;; perspective these are all "bad/unreachable issuer URL".
|
||||
(raise-invalid-sso-config
|
||||
:hint "unable to discover OIDC configuration"
|
||||
:discover-uri uri
|
||||
:cause cause))))))
|
||||
(-> provider
|
||||
(assoc :token-uri token-uri)
|
||||
(assoc :auth-uri auth-uri)
|
||||
(assoc :user-uri user-uri)
|
||||
(assoc :jwks-uri jwks-uri)
|
||||
(assoc :logout-uri logout-uri)))
|
||||
|
||||
(ex/raise :type ::internal
|
||||
:code :invalid-sso-config
|
||||
:hint "unable to discover OIDC configuration"
|
||||
:discover-uri uri
|
||||
:response-status-code (:status rsp)))))
|
||||
|
||||
(def ^:private default-oidc-scopes
|
||||
#{"openid" "profile" "email"})
|
||||
@@ -128,29 +107,16 @@
|
||||
|
||||
(defn- fetch-oidc-jwks
|
||||
[cfg jwks-uri {:keys [skip-ssrf-check?]}]
|
||||
(try
|
||||
(let [{:keys [status body]} (http/req cfg {:method :get :uri jwks-uri} {:skip-ssrf-check? skip-ssrf-check?})]
|
||||
(if (= 200 status)
|
||||
(-> body json/decode :keys process-oidc-jwks)
|
||||
(raise-invalid-sso-config
|
||||
:hint "unable to retrieve JWKs (unexpected response status code)"
|
||||
:jwks-uri jwks-uri
|
||||
:response-status-code status)))
|
||||
(catch Throwable cause
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw cause)
|
||||
(raise-invalid-sso-config
|
||||
:hint "unable to retrieve JWKs"
|
||||
:jwks-uri jwks-uri
|
||||
:cause cause)))))
|
||||
(let [{:keys [status body]} (http/req cfg {:method :get :uri jwks-uri} {:skip-ssrf-check? skip-ssrf-check?})]
|
||||
(if (= 200 status)
|
||||
(-> body json/decode :keys process-oidc-jwks)
|
||||
(ex/raise :type ::internal
|
||||
:code :unable-to-fetch-sso-jwks
|
||||
:hint "unable to retrieve JWKs (unexpected response status code)"
|
||||
:response-status-code status))))
|
||||
|
||||
(defn- populate-jwks
|
||||
"Fetch and add JWKs to the OIDC provider.
|
||||
|
||||
When `:strict-jwks?` is set (organization SSO), failures raise a controlled
|
||||
validation error. Otherwise JWKS is best-effort: log and continue without keys
|
||||
so global OIDC/GitLab providers can still initialize if JWKS is temporarily down."
|
||||
"Fetch and Add (if possible) JWK's to the OIDC provider"
|
||||
[cfg provider]
|
||||
(try
|
||||
(if-let [jwks (when-let [jwks-uri (:jwks-uri provider)]
|
||||
@@ -158,28 +124,20 @@
|
||||
(assoc provider :jwks jwks)
|
||||
provider)
|
||||
(catch Throwable cause
|
||||
(if (:strict-jwks? provider)
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw cause)
|
||||
(raise-invalid-sso-config
|
||||
:hint "unable to retrieve JWKs"
|
||||
:provider (:id provider)
|
||||
:cause cause))
|
||||
(do
|
||||
(l/warn :hint "unable to fetch JWKs for the OIDC provider"
|
||||
:provider (str (:id provider))
|
||||
:cause cause)
|
||||
provider)))))
|
||||
(l/warn :hint "unable to fetch JWKs for the OIDC provider"
|
||||
:provider (str (:id provider))
|
||||
:cause cause)
|
||||
provider)))
|
||||
|
||||
(defn- prepare-oidc-provider
|
||||
[cfg params]
|
||||
(when-not (and (string? (:base-uri params))
|
||||
(string? (:client-id params))
|
||||
(string? (:client-secret params)))
|
||||
(raise-invalid-sso-config
|
||||
:hint "missing params for provider initialization"
|
||||
:provider (:id params)))
|
||||
(ex/raise :type ::internal
|
||||
:code :invalid-sso-config
|
||||
:hint "missing params for provider initialization"
|
||||
:provider (:id params)))
|
||||
|
||||
(try
|
||||
(if (and (string? (:token-uri params))
|
||||
@@ -192,13 +150,11 @@
|
||||
(with-meta provider {::discovered true})))
|
||||
|
||||
(catch Throwable cause
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw cause)
|
||||
(raise-invalid-sso-config
|
||||
:hint "unexpected exception on configuring provider"
|
||||
:provider (:id params)
|
||||
:cause cause)))))
|
||||
(ex/raise :type ::internal
|
||||
:type :invalid-sso-config
|
||||
:hint "unexpected exception on configuring provider"
|
||||
:provider (:id params)
|
||||
:cause cause))))
|
||||
|
||||
(defmethod ig/assert-key ::providers/generic
|
||||
[_ params]
|
||||
@@ -366,9 +322,10 @@
|
||||
[cfg params]
|
||||
(when-not (and (string? (:client-id params))
|
||||
(string? (:client-secret params)))
|
||||
(raise-invalid-sso-config
|
||||
:hint "missing params for provider initialization"
|
||||
:provider (:id params)))
|
||||
(ex/raise :type ::internal
|
||||
:code :invalid-sso-config
|
||||
:hint "missing params for provider initialization"
|
||||
:provider (:id params)))
|
||||
|
||||
(try
|
||||
(let [provider (populate-jwks cfg params)]
|
||||
@@ -379,13 +336,11 @@
|
||||
:client-secret (d/obfuscate-string (:client-secret provider)))
|
||||
provider)
|
||||
(catch Throwable cause
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw cause)
|
||||
(raise-invalid-sso-config
|
||||
:hint "unexpected exception on configuring provider"
|
||||
:provider (:id params)
|
||||
:cause cause)))))
|
||||
(ex/raise :type ::internal
|
||||
:type :invalid-sso-config
|
||||
:hint "unexpected exception on configuring provider"
|
||||
:provider (:id params)
|
||||
:cause cause))))
|
||||
|
||||
(defmethod ig/init-key ::providers/gitlab
|
||||
[_ cfg]
|
||||
@@ -665,6 +620,9 @@
|
||||
(some? (:external-session-id state))
|
||||
(assoc :external-session-id (:external-session-id state))
|
||||
|
||||
(some? (:token/expires-in tdata))
|
||||
(assoc :sso-token-exp (ct/in-future {:seconds (:token/expires-in tdata)}))
|
||||
|
||||
;; If state token comes with props, merge them. The state token
|
||||
;; props can contain pm_ and utm_ prefixed query params.
|
||||
(map? (:props state))
|
||||
@@ -692,15 +650,6 @@
|
||||
(assoc :query (u/map->query-string params)))]
|
||||
(redirect-response uri))))
|
||||
|
||||
(defn- redirect-with-organization-sso-error
|
||||
[{:keys [dest-url organization-id organization-name]}]
|
||||
(-> (str (or dest-url (cf/get :public-uri)))
|
||||
(u/append-query-param :sso-error true)
|
||||
(u/append-query-param :organization-id organization-id)
|
||||
(cond-> organization-name
|
||||
(u/append-query-param :organization-name organization-name))
|
||||
(redirect-response)))
|
||||
|
||||
(defn- redirect-to-register
|
||||
[cfg info provider]
|
||||
(let [info (assoc info
|
||||
@@ -816,82 +765,6 @@
|
||||
;; ORG SSO HELPERS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- organization-sso-oauth-failure-reason
|
||||
[error]
|
||||
(case (d/name error)
|
||||
"access_denied" "access-denied"
|
||||
("temporarily_unavailable" "server_error") "provider-unavailable"
|
||||
("invalid_request" "unauthorized_client" "invalid_scope") "invalid-configuration"
|
||||
"provider-error"))
|
||||
|
||||
(defn- organization-sso-exception-failure-reason
|
||||
[cause]
|
||||
(let [data (ex-data cause)
|
||||
status (or (:response-status data)
|
||||
(:response-status-code data)
|
||||
(:http-status data))
|
||||
network-error?
|
||||
(loop [current cause]
|
||||
(cond
|
||||
(nil? current)
|
||||
false
|
||||
|
||||
(or (instance? java.net.ConnectException current)
|
||||
(instance? java.net.UnknownHostException current)
|
||||
(instance? java.net.http.HttpTimeoutException current)
|
||||
(instance? javax.net.ssl.SSLException current))
|
||||
true
|
||||
|
||||
(identical? current (ex-cause current))
|
||||
false
|
||||
|
||||
:else
|
||||
(recur (ex-cause current))))]
|
||||
(if (or network-error?
|
||||
(and (number? status) (<= 500 status 599)))
|
||||
"provider-unavailable"
|
||||
(case (:code data)
|
||||
:unable-to-fetch-access-token "token-exchange-failed"
|
||||
:unable-to-retrieve-user-info "user-info-failed"
|
||||
:incomplete-user-info "incomplete-user-info"
|
||||
:invalid-sso-config "invalid-configuration"
|
||||
:unable-to-fetch-sso-jwks "provider-unavailable"
|
||||
:unable-to-auth "access-denied"
|
||||
"unexpected-error"))))
|
||||
|
||||
(defn- submit-organization-sso-auth-event
|
||||
[cfg request profile-id organization-id name & {:keys [failure-reason]}]
|
||||
(audit/submit cfg {:type "action"
|
||||
:name name
|
||||
:profile-id profile-id
|
||||
:ip-addr (inet/parse-request request)
|
||||
:props (d/without-nils
|
||||
{:organization-id organization-id
|
||||
:failure-reason failure-reason})
|
||||
:context (audit/prepare-context-from-request request)}))
|
||||
|
||||
(defn submit-organization-sso-auth-started-event
|
||||
[cfg request profile-id organization-id]
|
||||
(submit-organization-sso-auth-event
|
||||
cfg request profile-id organization-id "organization-sso-auth-started"))
|
||||
|
||||
(defn submit-organization-sso-auth-failed-event
|
||||
[cfg request profile-id organization-id cause]
|
||||
(submit-organization-sso-auth-event
|
||||
cfg request profile-id organization-id "organization-sso-auth-failed"
|
||||
:failure-reason (organization-sso-exception-failure-reason cause)))
|
||||
|
||||
(defn- submit-organization-sso-oauth-failed-event
|
||||
[cfg request state-token error]
|
||||
(try
|
||||
(let [state (tokens/verify cfg {:token state-token :iss "oidc"})]
|
||||
(when (:dest-url state)
|
||||
(submit-organization-sso-auth-event
|
||||
cfg request (some-> (session/get-session request) :profile-id)
|
||||
(:organization-id state) "organization-sso-auth-failed"
|
||||
:failure-reason (organization-sso-oauth-failure-reason error))))
|
||||
(catch Exception _ nil)))
|
||||
|
||||
(defn- non-blank-uri
|
||||
[value]
|
||||
(when-not (str/blank? value) value))
|
||||
@@ -903,7 +776,7 @@
|
||||
|
||||
(defn prepare-organization-sso-provider
|
||||
"Build an OIDC provider map dynamically from the Nitrate organization SSO config.
|
||||
Uses OIDC discovery via :issuer when token/auth/user URIs are absent."
|
||||
Uses OIDC discovery via :issuer when token/auth/user URIs are absent."
|
||||
[cfg {:keys [client-id client-secret issuer]}]
|
||||
(prepare-oidc-provider cfg
|
||||
{:type "oidc"
|
||||
@@ -913,9 +786,7 @@
|
||||
(str/rtrim "/")
|
||||
(str "/"))
|
||||
:scopes default-oidc-scopes
|
||||
;; Organization SSO is configured by customers; discovery
|
||||
;; and JWKS failures must surface as controlled errors.
|
||||
:strict-jwks? true}))
|
||||
:skip-ssrf-check? true}))
|
||||
|
||||
(defn build-organization-sso-auth-redirect-uri
|
||||
"Build the OIDC authorization redirect URI for an organization SSO config.
|
||||
@@ -925,24 +796,16 @@
|
||||
issuer (organization-sso-discovery-uri sso)
|
||||
dest-url (or dest-url (str (cf/get :public-uri)))]
|
||||
(when-not issuer
|
||||
(raise-invalid-sso-config
|
||||
:hint "missing issuer"
|
||||
:organization-id organization-id))
|
||||
(try
|
||||
(let [oidc-provider (or provider (prepare-organization-sso-provider cfg sso))
|
||||
state-token (tokens/generate cfg {:iss "oidc"
|
||||
:dest-url dest-url
|
||||
:organization-id organization-id
|
||||
:issuer issuer
|
||||
:exp (ct/in-future "4h")})]
|
||||
(build-auth-redirect-uri oidc-provider state-token))
|
||||
(catch Throwable cause
|
||||
(if (and (ex/error? cause)
|
||||
(= :invalid-sso-config (:code (ex-data cause))))
|
||||
(throw (ex-info (ex-message cause)
|
||||
(assoc (ex-data cause) :organization-id organization-id)
|
||||
(ex-cause cause)))
|
||||
(throw cause))))))
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-sso-config
|
||||
:hint "missing issuer"))
|
||||
(let [oidc-provider (or provider (prepare-organization-sso-provider cfg sso))
|
||||
state-token (tokens/generate cfg {:iss "oidc"
|
||||
:dest-url dest-url
|
||||
:organization-id organization-id
|
||||
:issuer issuer
|
||||
:exp (ct/in-future "4h")})]
|
||||
(build-auth-redirect-uri oidc-provider state-token))))
|
||||
|
||||
(def ^:private probe-auth-code "penpot-sso-config-probe")
|
||||
|
||||
@@ -1025,53 +888,10 @@
|
||||
{::yres/status 200
|
||||
::yres/body {:redirect-uri uri}}))
|
||||
|
||||
(defn- organization-sso-callback-handler
|
||||
"Handle the organization-SSO branch of the OIDC callback: state carries
|
||||
:dest-url — exchange the authorization code with the OIDC provider to
|
||||
verify authentication actually occurred, then redirect back to dest-url."
|
||||
[cfg request state code]
|
||||
(let [dest-url (:dest-url state)]
|
||||
(try
|
||||
(let [organization-id (:organization-id state)
|
||||
sso (nitrate/call cfg :get-organization-sso {:organization-id organization-id})
|
||||
provider (prepare-organization-sso-provider cfg sso)
|
||||
_info (get-info cfg provider state code)
|
||||
session (session/get-session request)
|
||||
exp (ct/in-future {:minutes 15})]
|
||||
(when (and session organization-id)
|
||||
(let [props (-> (or (:props session) {})
|
||||
(update :sso assoc organization-id exp))]
|
||||
(session/update-session (::session/manager cfg) (assoc session :props props))))
|
||||
(submit-organization-sso-auth-event
|
||||
cfg request (:profile-id session) organization-id "organization-sso-auth-succeeded")
|
||||
(redirect-response dest-url))
|
||||
(catch Throwable cause
|
||||
(let [{:keys [code]} (ex-data cause)]
|
||||
(binding [l/*context* (errors/request->context request)]
|
||||
(if (some? code)
|
||||
(l/warn :hint "organization sso callback failed"
|
||||
:code code
|
||||
:message (ex-message cause)
|
||||
:organization-id (:organization-id state))
|
||||
(l/err :hint "unexpected error on organization sso callback"
|
||||
:organization-id (:organization-id state)
|
||||
:cause cause))))
|
||||
(submit-organization-sso-auth-failed-event
|
||||
cfg request (some-> (session/get-session request) :profile-id)
|
||||
(:organization-id state) cause)
|
||||
(let [organization-id (:organization-id state)
|
||||
organization-name (:name (nitrate/call cfg :get-organization-summary {:organization-id organization-id}))]
|
||||
(redirect-with-organization-sso-error
|
||||
{:dest-url dest-url
|
||||
:organization-id organization-id
|
||||
:organization-name organization-name}))))))
|
||||
|
||||
(defn- callback-handler
|
||||
[cfg {:keys [params] :as request}]
|
||||
(if-let [error (get params :error)]
|
||||
(do
|
||||
(submit-organization-sso-oauth-failed-event cfg request (:state params) error)
|
||||
(redirect-with-error "unable-to-auth" error))
|
||||
(redirect-with-error "unable-to-auth" error)
|
||||
(try
|
||||
(let [code (get params :code)
|
||||
state (get params :state)
|
||||
@@ -1079,8 +899,18 @@
|
||||
|
||||
;; Organization SSO flow: state carries :dest-url — exchange the authorization
|
||||
;; code with the OIDC provider to verify authentication actually occurred.
|
||||
(if (:dest-url state)
|
||||
(organization-sso-callback-handler cfg request state code)
|
||||
(if-let [dest-url (:dest-url state)]
|
||||
(let [organization-id (:organization-id state)
|
||||
sso (nitrate/call cfg :get-organization-sso {:organization-id organization-id})
|
||||
provider (prepare-organization-sso-provider cfg sso)
|
||||
info (get-info cfg provider state code)
|
||||
session (session/get-session request)
|
||||
exp (or (:sso-token-exp info) (ct/in-future {:hours 48}))]
|
||||
(when (and session organization-id)
|
||||
(let [props (-> (or (:props session) {})
|
||||
(update :sso assoc organization-id exp))]
|
||||
(session/update-session (::session/manager cfg) (assoc session :props props))))
|
||||
(redirect-response dest-url))
|
||||
|
||||
(let [provider (resolve-provider cfg state)
|
||||
info (get-info cfg provider state code)
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.auth.passwords
|
||||
"Password strength validation using Passay library."
|
||||
(:require
|
||||
[app.common.exceptions :as ex])
|
||||
(:import
|
||||
[org.passay CharacterCharacteristicsRule CharacterRule EnglishCharacterData PasswordData]))
|
||||
|
||||
(defonce ^:private passay-code->translation-key
|
||||
{"INSUFFICIENT_LOWERCASE" "errors.weak-password.insufficient-lowercase"
|
||||
"INSUFFICIENT_UPPERCASE" "errors.weak-password.insufficient-uppercase"
|
||||
"INSUFFICIENT_DIGIT" "errors.weak-password.insufficient-digits"
|
||||
"INSUFFICIENT_SPECIAL" "errors.weak-password.insufficient-special"})
|
||||
|
||||
(defonce ^:private character-characteristics-rule
|
||||
(doto (CharacterCharacteristicsRule.)
|
||||
(.setRules [(CharacterRule. EnglishCharacterData/LowerCase 1)
|
||||
(CharacterRule. EnglishCharacterData/UpperCase 1)
|
||||
(CharacterRule. EnglishCharacterData/Digit 1)
|
||||
(CharacterRule. EnglishCharacterData/Special 1)])
|
||||
(.setNumberOfCharacteristics 4)))
|
||||
|
||||
(defn validate-password
|
||||
"Validates password strength.
|
||||
Returns nil if valid, or raises exception if invalid.
|
||||
Checks:
|
||||
- Minimum length of 8 characters
|
||||
- At least 1 lowercase letter
|
||||
- At least 1 uppercase letter
|
||||
- At least 1 digit
|
||||
- At least 1 special character"
|
||||
[password]
|
||||
(when (< (count password) 8)
|
||||
(ex/raise :type :validation
|
||||
:code :weak-password
|
||||
:hint "password must be at least 8 characters"
|
||||
:details ["errors.weak-password.too-short"]))
|
||||
|
||||
(let [password-data (PasswordData. password)
|
||||
char-result (.validate character-characteristics-rule password-data)]
|
||||
(when-not (.isValid char-result)
|
||||
(ex/raise :type :validation
|
||||
:code :weak-password
|
||||
:hint "password must contain at least 1 lowercase letter, 1 uppercase letter, 1 digit, and 1 special character"
|
||||
:details (->> (.getDetails char-result)
|
||||
(mapv #(.getErrorCode %))
|
||||
(mapv passay-code->translation-key)
|
||||
(filterv some?))))))
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.cleaner
|
||||
"A collection of helpers for perform cleaning of artifacts; mainly
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.common
|
||||
"A binfile related file processing common code, used for different
|
||||
@@ -27,6 +27,7 @@
|
||||
[app.features.file-migrations :as fmigr]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.storage :as sto]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.worker :as-alias wrk]
|
||||
@@ -653,6 +654,27 @@
|
||||
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
|
||||
|
||||
(defn invalidate-thumbnails
|
||||
[cfg file-id]
|
||||
(let [storage (sto/resolve cfg)
|
||||
|
||||
sql-1
|
||||
(str "update file_tagged_object_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")
|
||||
|
||||
sql-2
|
||||
(str "update file_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")]
|
||||
|
||||
(run! #(sto/touch-object! storage %)
|
||||
(sequence
|
||||
(keep :media-id)
|
||||
(concat
|
||||
(db/exec! cfg [sql-1 file-id])
|
||||
(db/exec! cfg [sql-2 file-id]))))))
|
||||
|
||||
(defn process-file
|
||||
[cfg {:keys [id] :as file}]
|
||||
(let [libs (delay (get-resolved-file-libraries cfg file))]
|
||||
@@ -701,7 +723,6 @@
|
||||
(-> (select-keys file file-attrs)
|
||||
(assoc :data nil)
|
||||
(dissoc :team-id)
|
||||
(dissoc :metadata)
|
||||
(dissoc :migrations)))
|
||||
|
||||
(defn- file->file-data-params
|
||||
@@ -727,17 +748,9 @@
|
||||
(fmigr/upsert-migrations! conn file))
|
||||
|
||||
(let [file (encode-file cfg file)]
|
||||
(try
|
||||
(db/insert! conn :file
|
||||
(file->params file)
|
||||
(assoc opts ::db/return-keys false))
|
||||
(catch org.postgresql.util.PSQLException cause
|
||||
(if (db/duplicate-key-error? cause)
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "file already exists"
|
||||
:cause cause)
|
||||
(throw cause))))
|
||||
(db/insert! conn :file
|
||||
(file->params file)
|
||||
(assoc opts ::db/return-keys false))
|
||||
|
||||
(->> (file->file-data-params file)
|
||||
(fdata/upsert! cfg))
|
||||
@@ -853,8 +866,8 @@
|
||||
(defn get-resolved-file-libraries
|
||||
"Get all file libraries including itself. Returns an instance of
|
||||
LoadableWeakValueMap that allows do not have strong references to
|
||||
the loaded libraries and reduce memory pressure on having
|
||||
all this libraries at the same time on processing file validation
|
||||
the loaded libraries and reduce possible memory pressure on having
|
||||
all this libraries loaded at same time on processing file validation
|
||||
or file migration.
|
||||
|
||||
This still requires at least one library at time to be loaded while
|
||||
@@ -866,47 +879,3 @@
|
||||
(cons (:id file)))
|
||||
load-fn #(get-file cfg % :migrate? false)]
|
||||
(weak/loadable-weak-value-map library-ids load-fn {id file})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; EXTERNAL LIBRARY RESOLUTION HELPERS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn slugify-name
|
||||
"Slugify a library name for cross-environment matching.
|
||||
Lowercases, replaces non-alphanumeric runs with '-', strips
|
||||
leading/trailing '-'."
|
||||
[name]
|
||||
(str/slug name))
|
||||
|
||||
(def ^:private sql:get-files-names
|
||||
"SELECT id, name FROM file WHERE id = ANY(?)")
|
||||
|
||||
(defn get-files-names
|
||||
"Return [{:id uuid :name string}] for the given file ids."
|
||||
[cfg ids]
|
||||
(db/run! cfg
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(let [ids-arr (db/create-array conn "uuid" ids)]
|
||||
(db/exec! conn [sql:get-files-names ids-arr])))))
|
||||
|
||||
(def ^:private sql:get-shared-files-for-team
|
||||
"SELECT f.id, f.name, f.project_id
|
||||
FROM file AS f
|
||||
JOIN project AS p ON (p.id = f.project_id)
|
||||
WHERE p.team_id = ?
|
||||
AND f.is_shared = true
|
||||
AND f.deleted_at IS NULL
|
||||
AND p.deleted_at IS NULL")
|
||||
|
||||
(defn get-shared-files-for-team
|
||||
"Return [{:id uuid :name string}] for all shared files in a team."
|
||||
[cfg team-id]
|
||||
(db/run! cfg
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(db/exec! conn [sql:get-shared-files-for-team team-id]))))
|
||||
|
||||
(defn find-shared-files-by-slug
|
||||
"Return all shared files in `team-id` whose slugified name equals `slug`."
|
||||
[cfg team-id slug]
|
||||
(->> (get-shared-files-for-team cfg team-id)
|
||||
(filter #(= slug (slugify-name (:name %))))))
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.migrations
|
||||
"A binfile related migrations handling"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.v1
|
||||
"A custom, perfromance and efficiency focused binfile format impl"
|
||||
@@ -174,10 +174,6 @@
|
||||
(assert-mark m :obj)
|
||||
(let [size (read-long! input)]
|
||||
(assert (pos? size) "incorrect header size found on reading header")
|
||||
(when (> size bfc/max-object-size)
|
||||
(ex/raise :type :validation
|
||||
:code :max-file-size-reached
|
||||
:hint (dm/str "unable to import object with size " size " bytes")))
|
||||
(let [buff (byte-array size)]
|
||||
(read-bytes! input buff)
|
||||
(fres/decode buff)))))
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.v2
|
||||
"A sqlite3 based binary file exportation with support for exportation
|
||||
|
||||
+38
-217
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.binfile.v3
|
||||
"A ZIP based binary file exportation"
|
||||
@@ -67,16 +67,7 @@
|
||||
|
||||
[:relations {:optional true}
|
||||
[:vector
|
||||
[:tuple ::sm/uuid ::sm/uuid]]]
|
||||
|
||||
;; TODO: rename to :links
|
||||
[:external-libraries {:optional true}
|
||||
[:vector
|
||||
[:map
|
||||
[:id ::sm/uuid]
|
||||
[:name :string]
|
||||
[:slug :string]
|
||||
[:used-by {:optional true} [:vector ::sm/uuid]]]]]])
|
||||
[:tuple ::sm/uuid ::sm/uuid]]]])
|
||||
|
||||
(def ^:private schema:storage-object
|
||||
[:map {:title "StorageObject"}
|
||||
@@ -226,12 +217,14 @@
|
||||
(.flush writer))
|
||||
(.closeEntry output))
|
||||
|
||||
|
||||
(defn- get-file
|
||||
[{:keys [::bfc/export-type] :as cfg} file-id]
|
||||
[{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id]
|
||||
|
||||
(let [detach? (= export-type :detach-libraries)
|
||||
embed? (= export-type :merge-libraries)]
|
||||
(when (and include-libraries embed-assets)
|
||||
(throw (IllegalArgumentException.
|
||||
"the `include-libraries` and `embed-assets` are mutally excluding options")))
|
||||
|
||||
(let [detach? (and (not embed-assets) (not include-libraries))]
|
||||
(db/tx-run! cfg (fn [cfg]
|
||||
(cond-> (bfc/get-file cfg file-id
|
||||
{:realize? true
|
||||
@@ -241,7 +234,7 @@
|
||||
(-> (ctf/detach-external-references file-id)
|
||||
(dissoc :libraries))
|
||||
|
||||
embed?
|
||||
embed-assets
|
||||
(update :data #(bfc/embed-assets cfg % file-id))
|
||||
|
||||
:always
|
||||
@@ -378,34 +371,12 @@
|
||||
(write-entry! output path encoded-tokens)))))
|
||||
|
||||
(defn- export-files
|
||||
[{:keys [::bfc/ids ::bfc/export-type ::output] :as cfg}]
|
||||
|
||||
(let [original-ids ids
|
||||
ids (into ids (when (= export-type :include-libraries) (bfc/get-libraries cfg ids)))
|
||||
rels (if (= export-type :include-libraries)
|
||||
[{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}]
|
||||
(let [ids (into ids (when include-libraries (bfc/get-libraries cfg ids)))
|
||||
rels (if include-libraries
|
||||
(->> (bfc/get-files-rels cfg ids)
|
||||
(mapv (juxt :file-id :library-file-id)))
|
||||
[])
|
||||
|
||||
;; Compute external libraries: referenced by original files but
|
||||
;; not included in the export set. Only relevant for :link-later.
|
||||
external-libs
|
||||
(when (= export-type :link-later)
|
||||
(let [original-rels (bfc/get-files-rels cfg original-ids)
|
||||
lib-ids (into #{} (map :library-file-id) original-rels)]
|
||||
(when (seq lib-ids)
|
||||
(let [lib-names (bfc/get-files-names cfg lib-ids)]
|
||||
(->> lib-names
|
||||
(mapv (fn [{:keys [id name]}]
|
||||
(let [slug (bfc/slugify-name name)]
|
||||
(when-not (str/blank? slug)
|
||||
{:id id
|
||||
:name name
|
||||
:slug slug
|
||||
:used-by (->> original-rels
|
||||
(filter #(= (:library-file-id %) id))
|
||||
(mapv :file-id))}))))
|
||||
(filterv some?))))))]
|
||||
[])]
|
||||
|
||||
(vswap! bfc/*state* assoc :files (d/ordered-map))
|
||||
|
||||
@@ -418,14 +389,12 @@
|
||||
|
||||
;; Write manifest file
|
||||
(let [files (:files @bfc/*state*)
|
||||
params (cond-> {:type "penpot/export-files"
|
||||
:version 1
|
||||
:generated-by (str "penpot/" (:full cf/version))
|
||||
:referer "penpot"
|
||||
:files (vec (vals files))
|
||||
:relations rels}
|
||||
(seq external-libs)
|
||||
(assoc :external-libraries external-libs))]
|
||||
params {:type "penpot/export-files"
|
||||
:version 1
|
||||
:generated-by (str "penpot/" (:full cf/version))
|
||||
:refer "penpot"
|
||||
:files (vec (vals files))
|
||||
:relations rels}]
|
||||
(write-entry! output "manifest.json" params))))
|
||||
|
||||
;; --- IMPORT IMPL
|
||||
@@ -765,7 +734,7 @@
|
||||
:plugin-data plugin-data}))
|
||||
|
||||
(defn- import-file
|
||||
[{:keys [::db/conn ::bfc/project-id ::manifest] :as cfg} {file-id :id file-name :name}]
|
||||
[{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
|
||||
(let [file-id' (bfc/lookup-index file-id)
|
||||
file (read-file cfg file-id)
|
||||
media (read-file-media cfg file-id)
|
||||
@@ -832,10 +801,8 @@
|
||||
(assoc :data data)
|
||||
(assoc :name file-name)
|
||||
(assoc :project-id project-id)
|
||||
(assoc :metadata (d/without-nils
|
||||
{:generated-by (get manifest :generated-by)
|
||||
:referer (or (get manifest :referer) (get manifest :refer))}))
|
||||
(dissoc :options))
|
||||
|
||||
file (bfc/process-file cfg file)
|
||||
file (ctf/check-file file)]
|
||||
|
||||
@@ -866,13 +833,6 @@
|
||||
[{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}]
|
||||
(events/tap :progress {:section :storage-objects})
|
||||
|
||||
;; IMPORTANT: we strongly do not reuse the main connection that can
|
||||
;; run inside a transaction because the storage upload process can
|
||||
;; fail in the middle of uploading and leave garbage on the underlying
|
||||
;; backend, if we participate in the main transaction and it aborts
|
||||
;; we will lose all registry of the pending to reconcile blobs
|
||||
;; what the storage subsystem registers in other parallel
|
||||
;; transaction
|
||||
(let [storage (sto/resolve cfg)
|
||||
entries (keep (match-storage-entry-fn) entries)]
|
||||
|
||||
@@ -920,104 +880,6 @@
|
||||
|
||||
(vswap! bfc/*state* update :index assoc id (:id sobject)))))))
|
||||
|
||||
(defn- add-to-file
|
||||
"Add a resolved library entry to a file in the file-grouped resolution.
|
||||
`key` is :done (auto-linked) or :pending (needs resolution)."
|
||||
[acc file-id file-name key entry]
|
||||
(update acc file-id (fn [file]
|
||||
(let [file (or file {:id file-id
|
||||
:name file-name
|
||||
:done []
|
||||
:pending []})]
|
||||
(update file key conj entry)))))
|
||||
|
||||
(defn- compute-link-decisions
|
||||
"Returns a map of {old-lib-id -> {:library-id ... :library ...}} for external
|
||||
libraries that should be auto-linked (single candidate AND importer has edit
|
||||
permission). Libraries with zero or multiple candidates, or where the importer
|
||||
lacks permission, are excluded — their refs should remain dangling."
|
||||
[{:keys [::db/conn ::manifest ::bfc/team-id ::bfc/profile-id] :as cfg}]
|
||||
(reduce
|
||||
(fn [acc ext-lib]
|
||||
(let [slug (:slug ext-lib)]
|
||||
(if (nil? slug)
|
||||
acc
|
||||
(let [matching (into [] (bfc/find-shared-files-by-slug cfg team-id slug))]
|
||||
(if (not= 1 (count matching))
|
||||
acc
|
||||
(let [library (first matching)
|
||||
perms (bfc/get-file-permissions conn profile-id (:id library))]
|
||||
(if (:can-edit perms)
|
||||
(assoc acc (:id ext-lib) {:library-id (:id library)
|
||||
:library library})
|
||||
acc)))))))
|
||||
{}
|
||||
(:external-libraries manifest)))
|
||||
|
||||
(defn- resolve-and-link-libraries
|
||||
"For each external library in the manifest, resolve candidates by slug.
|
||||
Auto-links single matches (creating DB rows) and builds a file-grouped
|
||||
resolution map keyed by imported file-id (new UUID)."
|
||||
|
||||
[{:keys [::db/conn ::manifest ::bfc/team-id ::bfc/timestamp] :as cfg} files-info]
|
||||
(assert (uuid? team-id) "team-id should be provided")
|
||||
|
||||
(let [file-ids (keys files-info)
|
||||
decisions (compute-link-decisions cfg)]
|
||||
|
||||
(reduce
|
||||
(fn [acc ext-lib]
|
||||
(assert (contains? ext-lib :id) "expected `:id` on ext-lib")
|
||||
(assert (contains? ext-lib :name) "expected `:name` on ext-lib")
|
||||
(assert (contains? ext-lib :used-by) "expected `:used-by` on ext-lib")
|
||||
(assert (contains? ext-lib :slug) "expected `:slug` on ext-lib")
|
||||
|
||||
(let [used-by (into #{} (map bfc/lookup-index) (:used-by ext-lib))]
|
||||
(cond
|
||||
;; No slug → skip
|
||||
(nil? (:slug ext-lib))
|
||||
acc
|
||||
|
||||
;; Has decision → auto-link (single match + can-edit)
|
||||
(contains? decisions (:id ext-lib))
|
||||
(let [{:keys [library-id]} (get decisions (:id ext-lib))
|
||||
used-by (filter used-by file-ids)]
|
||||
(doseq [file-id used-by]
|
||||
(let [rel-params {:file-id file-id :library-file-id library-id}]
|
||||
(db/insert! conn :file-library-rel rel-params
|
||||
{::db/on-conflict-do-nothing? true})
|
||||
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))
|
||||
(let [entry {:id (:id ext-lib)
|
||||
:name (:name ext-lib)
|
||||
:linked-to library-id}]
|
||||
(reduce (fn [acc file-id]
|
||||
(add-to-file acc file-id (get files-info file-id) :done entry))
|
||||
acc used-by)))
|
||||
|
||||
;; Has candidates but no decision → multi-match or no permission → pending
|
||||
:else
|
||||
(let [matching-libraries (into [] (bfc/find-shared-files-by-slug cfg team-id (:slug ext-lib)))]
|
||||
(if (empty? matching-libraries)
|
||||
acc
|
||||
(let [candidates (mapv (fn [lib]
|
||||
(let [project-id (:project-id lib)
|
||||
project (bfc/get-project cfg project-id)
|
||||
project-name (:name project)]
|
||||
{:id (:id lib)
|
||||
:name (:name lib)
|
||||
:project-id project-id
|
||||
:project-name project-name}))
|
||||
matching-libraries)
|
||||
entry {:id (:id ext-lib)
|
||||
:name (:name ext-lib)
|
||||
:candidates candidates}]
|
||||
(reduce (fn [acc file-id]
|
||||
(add-to-file acc file-id (get files-info file-id) :pending entry))
|
||||
acc used-by)))))))
|
||||
|
||||
{}
|
||||
(:external-libraries manifest))))
|
||||
|
||||
(defn- import-files*
|
||||
[{:keys [::manifest] :as cfg}]
|
||||
(bfc/disable-database-timeouts! cfg)
|
||||
@@ -1026,58 +888,18 @@
|
||||
|
||||
(import-storage-objects cfg)
|
||||
|
||||
;; Pre-resolve external libraries and add their id mappings to the index
|
||||
;; BEFORE importing files. This allows relink-refs (inside process-file)
|
||||
;; to correctly remap :component-file references to the destination library.
|
||||
;; Only remap when a link will actually be created (single match + can-edit).
|
||||
(let [decisions (compute-link-decisions cfg)]
|
||||
(doseq [[old-lib-id {:keys [library-id]}] decisions]
|
||||
(l/trc :hint "pre-resolving external library"
|
||||
:old-id (str old-lib-id)
|
||||
:new-id (str library-id))
|
||||
(vswap! bfc/*state* update :index assoc old-lib-id library-id)))
|
||||
|
||||
(let [files (get manifest :files)
|
||||
file-ids (reduce (fn [result file]
|
||||
(let [name' (get file :name)
|
||||
file (assoc file :name name')]
|
||||
(conj result (import-file cfg file))))
|
||||
[]
|
||||
files)
|
||||
;; Build map of file-id to file-name for resolution
|
||||
files-info (into {} (map (fn [file-id manifest-file]
|
||||
[file-id (:name manifest-file)])
|
||||
file-ids
|
||||
files))]
|
||||
(let [files (get manifest :files)
|
||||
result (reduce (fn [result file]
|
||||
(let [name' (get file :name)
|
||||
file (assoc file :name name')]
|
||||
(conj result (import-file cfg file))))
|
||||
[]
|
||||
files)]
|
||||
|
||||
(import-file-relations cfg)
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
|
||||
(let [resolution (resolve-and-link-libraries cfg files-info)]
|
||||
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
{:file-ids file-ids
|
||||
:resolution resolution})))
|
||||
|
||||
(defn- invalidate-thumbnails
|
||||
[cfg file-id]
|
||||
(let [storage (sto/resolve cfg ::db/reuse-conn true)
|
||||
|
||||
sql-1
|
||||
(str "update file_tagged_object_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")
|
||||
|
||||
sql-2
|
||||
(str "update file_thumbnail "
|
||||
" set deleted_at = now() "
|
||||
" where file_id=? returning media_id")]
|
||||
|
||||
(run! #(sto/touch-object! storage %)
|
||||
(sequence
|
||||
(keep :media-id)
|
||||
(concat
|
||||
(db/exec! cfg [sql-1 file-id])
|
||||
(db/exec! cfg [sql-2 file-id]))))))
|
||||
result))
|
||||
|
||||
(defn- import-file-and-overwrite*
|
||||
[{:keys [::manifest ::bfc/file-id] :as cfg}]
|
||||
@@ -1102,11 +924,10 @@
|
||||
(import-storage-objects cfg)
|
||||
(import-file cfg file)
|
||||
|
||||
(invalidate-thumbnails cfg file-id)
|
||||
(bfc/invalidate-thumbnails cfg file-id)
|
||||
(bfm/apply-pending-migrations! cfg)
|
||||
|
||||
{:file-ids [file-id]
|
||||
:resolution {}})))
|
||||
[file-id])))
|
||||
|
||||
(defn- import-files
|
||||
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
|
||||
@@ -1154,11 +975,12 @@
|
||||
"Do the exportation of a specified file in custom penpot binary
|
||||
format. There are some options available for customize the output:
|
||||
|
||||
`::bfc/export-type`: determines how linked libraries are handled.
|
||||
Valid values: `:include-libraries` (include linked libraries),
|
||||
`:merge-libraries` (embed library assets in the file),
|
||||
`:detach-libraries` (treat assets as basic objects),
|
||||
`:link-later` (preserve component metadata for relinking on import)."
|
||||
`::bfc/include-libraries`: additionally to the specified file, all the
|
||||
linked libraries also will be included (including transitive
|
||||
dependencies).
|
||||
|
||||
`::bfc/embed-assets`: instead of including the libraries, embed in the
|
||||
same file library all assets used from external libraries."
|
||||
|
||||
[{:keys [::bfc/ids] :as cfg} output]
|
||||
|
||||
@@ -1174,7 +996,6 @@
|
||||
tp (ct/tpoint)
|
||||
ab (volatile! false)
|
||||
cs (volatile! nil)]
|
||||
|
||||
(try
|
||||
(l/info :hint "start exportation" :export-id (str id))
|
||||
(binding [bfc/*state* (volatile! (bfc/initial-state))]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.config
|
||||
(:refer-clojure :exclude [get])
|
||||
@@ -52,7 +52,7 @@
|
||||
|
||||
:redis-uri "redis://redis/0"
|
||||
|
||||
:file-data-backend "db"
|
||||
:file-data-backend "legacy-db"
|
||||
|
||||
:objects-storage-backend "fs"
|
||||
:objects-storage-fs-directory "assets"
|
||||
@@ -69,9 +69,6 @@
|
||||
:profile-bounce-max-age (ct/duration {:days 7})
|
||||
:profile-bounce-threshold 10
|
||||
|
||||
:login-lockout-max-attempts 5
|
||||
:login-lockout-window (ct/duration "15m")
|
||||
|
||||
:telemetry-uri "https://telemetry.penpot.app/"
|
||||
|
||||
:media-max-file-size (* 1024 1024 30) ; 30MiB
|
||||
@@ -154,9 +151,6 @@
|
||||
[:media-processing-service-uri {:optional true} ::sm/uri]
|
||||
[:media-processing-service-timeout {:optional true} ::sm/int]
|
||||
|
||||
[:login-lockout-max-attempts {:optional true} ::sm/int]
|
||||
[:login-lockout-window {:optional true} ::ct/duration]
|
||||
|
||||
[:deletion-delay {:optional true} ::ct/duration]
|
||||
[:file-clean-delay {:optional true} ::ct/duration]
|
||||
[:telemetry-enabled {:optional true} ::sm/boolean]
|
||||
@@ -200,7 +194,6 @@
|
||||
[:quotes-team-access-requests-per-requester {:optional true} ::sm/int]
|
||||
[:quotes-upload-sessions-per-profile {:optional true} ::sm/int]
|
||||
[:quotes-upload-chunks-per-session {:optional true} ::sm/int]
|
||||
[:quotes-media-storage-bytes-per-team {:optional true} ::sm/int]
|
||||
|
||||
[:auth-token-cookie-name {:optional true} :string]
|
||||
[:auth-token-cookie-max-age {:optional true} ::ct/duration]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.db
|
||||
(:refer-clojure :exclude [get run!])
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.db.sql
|
||||
(:refer-clojure :exclude [update])
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.email
|
||||
"Main api for send emails."
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.email.blacklist
|
||||
"Email blacklist provider"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.email.whitelist
|
||||
"Email whitelist provider"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.features.fdata
|
||||
"A `fdata/*` related feature migration helpers"
|
||||
@@ -12,7 +12,6 @@
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.objects-map :as omap]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
@@ -151,13 +150,6 @@
|
||||
|
||||
(cond
|
||||
(= backend "storage")
|
||||
;; IMPORTANT: we strongly do not reuse the main connection that can
|
||||
;; run inside a transaction because the storage upload process can
|
||||
;; fail in the middle of uploading and leave garbage on the underlying
|
||||
;; backend, if we participate in the main transaction and it aborts
|
||||
;; we will lose all registry of the pending to reconcile blobs
|
||||
;; what the storage subsystem registers in other parallel
|
||||
;; transaction
|
||||
(let [storage (sto/resolve cfg)
|
||||
content (sto/content data)
|
||||
sobject (sto/put-object! storage
|
||||
@@ -167,17 +159,15 @@
|
||||
:content-type "application/octet-stream"
|
||||
:file-id file-id
|
||||
:id id})
|
||||
metadata (-> (:metadata params)
|
||||
(assoc :storage-ref-id (:id sobject)))
|
||||
metadata {:storage-ref-id (:id sobject)}
|
||||
params (-> params
|
||||
(assoc :metadata metadata)
|
||||
(assoc :data nil))]
|
||||
(upsert-in-database cfg params))
|
||||
|
||||
(= backend "db")
|
||||
(let [metadata (dissoc (:metadata params) :storage-ref-id)
|
||||
params (assoc params :metadata metadata)]
|
||||
(upsert-in-database cfg params))
|
||||
(->> (dissoc params :metadata)
|
||||
(upsert-in-database cfg))
|
||||
|
||||
(= backend "legacy-db")
|
||||
(cond
|
||||
@@ -223,11 +213,18 @@
|
||||
[backend]
|
||||
(or backend (cf/get :file-data-backend)))
|
||||
|
||||
(def ^:private schema:metadata
|
||||
[:map {:title "Metadata"}
|
||||
[:storage-ref-id {:optional true} ::sm/uuid]])
|
||||
|
||||
(def decode-metadata-with-schema
|
||||
(sm/decoder schema:metadata sm/json-transformer))
|
||||
|
||||
(defn decode-metadata
|
||||
[metadata]
|
||||
(some-> metadata
|
||||
(db/decode-json-pgobject)
|
||||
(ctf/decode-file-metadata)))
|
||||
(decode-metadata-with-schema)))
|
||||
|
||||
(def ^:private schema:update-params
|
||||
[:map {:closed true}
|
||||
@@ -235,7 +232,7 @@
|
||||
[:type [:enum "main" "snapshot" "fragment"]]
|
||||
[:file-id ::sm/uuid]
|
||||
[:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
|
||||
[:metadata {:optional true} ctf/schema:file-metadata]
|
||||
[:metadata {:optional true} [:maybe schema:metadata]]
|
||||
[:data {:optional true} bytes?]
|
||||
[:created-at {:optional true} ::ct/inst]
|
||||
[:modified-at {:optional true} [:maybe ::ct/inst]]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.features.file-migrations
|
||||
"Backend specific code for file migrations. Implemented as permanent feature of files."
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.features.file-snapshots
|
||||
(:require
|
||||
@@ -326,11 +326,8 @@
|
||||
(let [file (d/update-when row :metadata fdata/decode-metadata)
|
||||
vern (rand-int Integer/MAX_VALUE)
|
||||
|
||||
;; We reuse the main connection here for storage operations
|
||||
;; becaue the main operations are touching and we need them
|
||||
;; to be atomic with the current transaction
|
||||
storage
|
||||
(sto/resolve cfg ::db/reuse-conn true)
|
||||
(sto/resolve cfg {::db/reuse-conn true})
|
||||
|
||||
snapshot
|
||||
(get-snapshot cfg file-id snapshot-id)]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.features.logical-deletion
|
||||
"A code related to handle logical deletion mechanism"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http
|
||||
(:require
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.access-token
|
||||
(:require
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.assets
|
||||
"Assets related handlers."
|
||||
@@ -14,7 +14,6 @@
|
||||
[app.db :as db]
|
||||
[app.http.access-token :as actoken]
|
||||
[app.http.session :as session]
|
||||
[app.rpc.permissions :as perms]
|
||||
[app.storage :as sto]
|
||||
[integrant.core :as ig]
|
||||
[yetti.response :as-alias yres]))
|
||||
@@ -41,38 +40,20 @@
|
||||
(ex/raise :type :not-found
|
||||
:hint "object not found")))
|
||||
|
||||
(defn- get-share-id
|
||||
"Extract and validate the optional `share-id` query param. Returns a UUID
|
||||
or `nil` for missing/malformed values."
|
||||
[{:keys [query-params]}]
|
||||
(some-> query-params :share-id d/parse-uuid))
|
||||
|
||||
(defn- get-file-media-object
|
||||
[pool id]
|
||||
(db/get* pool :file-media-object {:id id} {::db/remove-deleted false}))
|
||||
(db/get pool :file-media-object {:id id} {::db/remove-deleted false}))
|
||||
|
||||
(defn- serve-object-from-s3
|
||||
[{:keys [::sto/storage ::signature-max-age ::cache-max-age] :as cfg} obj]
|
||||
(let [sig-max-age (or signature-max-age default-signature-max-age)
|
||||
cch-max-age (or cache-max-age default-cache-max-age)
|
||||
bucket (-> obj meta :bucket)
|
||||
public? (contains? public-buckets bucket)
|
||||
;; The disposition is also signed into the presigned url: this
|
||||
;; response is a redirect, so the header below applies to the
|
||||
;; redirect itself and not to the bytes the client then fetches
|
||||
;; from the object store.
|
||||
{:keys [host port] :as url} (sto/get-object-url storage obj
|
||||
(cond-> {:max-age sig-max-age}
|
||||
(not public?)
|
||||
(assoc :content-disposition "attachment")))
|
||||
headers (cond-> {"location" (str url)
|
||||
"x-host" (cond-> host port (str ":" port))
|
||||
"x-mtype" (-> obj meta :content-type)
|
||||
"cache-control" (str "max-age=" (inst-ms cch-max-age))}
|
||||
(not public?)
|
||||
(assoc "content-disposition" "attachment"))]
|
||||
{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age sig-max-age})]
|
||||
{::yres/status 307
|
||||
::yres/headers headers}))
|
||||
::yres/headers {"location" (str url)
|
||||
"x-host" (cond-> host port (str ":" port))
|
||||
"x-mtype" (-> obj meta :content-type)
|
||||
"cache-control" (str "max-age=" (inst-ms cch-max-age))}}))
|
||||
|
||||
(defn- serve-object-from-fs
|
||||
[{:keys [::path ::cache-max-age]} obj]
|
||||
@@ -80,12 +61,9 @@
|
||||
purl (u/join (u/uri path)
|
||||
(sto/object->relative-path obj))
|
||||
mdata (meta obj)
|
||||
bucket (:bucket mdata)
|
||||
headers (cond-> {"x-accel-redirect" (:path purl)
|
||||
"content-type" (:content-type mdata)
|
||||
"cache-control" (str "max-age=" (inst-ms cch-max-age))}
|
||||
(not (contains? public-buckets bucket))
|
||||
(assoc "content-disposition" "attachment"))]
|
||||
headers {"x-accel-redirect" (:path purl)
|
||||
"content-type" (:content-type mdata)
|
||||
"cache-control" (str "max-age=" (inst-ms cch-max-age))}]
|
||||
{::yres/status 204
|
||||
::yres/headers headers}))
|
||||
|
||||
@@ -103,32 +81,17 @@
|
||||
(let [bucket (-> obj meta :bucket)]
|
||||
(not (contains? public-buckets bucket))))
|
||||
|
||||
(defn- request-profile-id
|
||||
"Extract the authenticated profile-id from the request."
|
||||
[request]
|
||||
(or (::session/profile-id request)
|
||||
(::actoken/profile-id request)))
|
||||
|
||||
(defn- authenticated?
|
||||
"Check if the request has an authenticated profile, either via session
|
||||
or access token."
|
||||
[request]
|
||||
(some? (request-profile-id request)))
|
||||
|
||||
(defn- tempfile-owner-match?
|
||||
"Check if the request's profile-id matches the tempfile's stored owner.
|
||||
Returns true if no profile-id was stored (legacy objects)."
|
||||
[obj request]
|
||||
(let [stored-profile-id (:profile-id (meta obj))
|
||||
request-profile-id (request-profile-id request)]
|
||||
(or (nil? stored-profile-id)
|
||||
(= stored-profile-id request-profile-id))))
|
||||
(or (some? (::session/profile-id request))
|
||||
(some? (::actoken/profile-id request))))
|
||||
|
||||
(defn objects-handler
|
||||
"Handler that serves storage objects by id.
|
||||
For non-public buckets (e.g. profile), requires authentication
|
||||
via session cookie or access token.
|
||||
For tempfile bucket, also requires ownership (profile-id match)."
|
||||
via session cookie or access token."
|
||||
[{:keys [::sto/storage] :as cfg} request]
|
||||
(let [id (get-id request)
|
||||
obj (sto/get-object storage id)]
|
||||
@@ -140,32 +103,19 @@
|
||||
(not (authenticated? request)))
|
||||
{::yres/status 401}
|
||||
|
||||
(and (= (-> obj meta :bucket) sto/tempfile-bucket)
|
||||
(not (tempfile-owner-match? obj request)))
|
||||
{::yres/status 404}
|
||||
|
||||
:else
|
||||
(serve-object cfg obj))))
|
||||
|
||||
(defn- generic-handler
|
||||
"A generic handler helper/common code for file-media based handlers."
|
||||
[{:keys [::sto/storage] :as cfg} request kf]
|
||||
(let [pool (::db/pool storage)
|
||||
id (get-id request)
|
||||
mobj (get-file-media-object pool id)]
|
||||
(if (nil? mobj)
|
||||
{::yres/status 404}
|
||||
(let [file-id (:file-id mobj)
|
||||
profile-id (or (::session/profile-id request)
|
||||
(::actoken/profile-id request))
|
||||
share-id (get-share-id request)
|
||||
perms (perms/get-file-read-permissions pool profile-id file-id share-id)]
|
||||
(if-not (:can-read perms)
|
||||
{::yres/status 404}
|
||||
(let [sobj (sto/get-object storage (kf mobj))]
|
||||
(if sobj
|
||||
(serve-object cfg sobj)
|
||||
{::yres/status 404})))))))
|
||||
(let [pool (::db/pool storage)
|
||||
id (get-id request)
|
||||
mobj (get-file-media-object pool id)
|
||||
sobj (sto/get-object storage (kf mobj))]
|
||||
(if sobj
|
||||
(serve-object cfg sobj)
|
||||
{::yres/status 404})))
|
||||
|
||||
(defn file-objects-handler
|
||||
"Handler that serves storage objects by file media id."
|
||||
|
||||
+184
-12
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.awsns
|
||||
"AWS SNS webhook handler for bounces."
|
||||
@@ -21,13 +21,158 @@
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[yetti.request :as yreq]
|
||||
[yetti.response :as-alias yres]))
|
||||
[yetti.response :as-alias yres])
|
||||
(:import
|
||||
java.net.URI
|
||||
java.security.cert.CertificateFactory
|
||||
java.security.Signature
|
||||
java.util.Base64))
|
||||
|
||||
(declare parse-json)
|
||||
(declare handle-request)
|
||||
(declare parse-notification)
|
||||
(declare process-report)
|
||||
|
||||
(defn- valid-sns-url?
|
||||
"Validates that a URL originates from an SNS endpoint.
|
||||
Only accepts sns.<region>.amazonaws.com hosts.
|
||||
See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html"
|
||||
[url]
|
||||
(when (string? url)
|
||||
(try
|
||||
(let [uri (URI. url)
|
||||
host (.getHost uri)]
|
||||
(and (= "https" (.getScheme uri))
|
||||
(boolean
|
||||
(re-matches
|
||||
#"(?i)sns\.[a-z0-9-]+\.amazonaws\.com"
|
||||
host))))
|
||||
(catch Exception _
|
||||
false))))
|
||||
|
||||
;; AWS SNS Signature Verification Field Sets
|
||||
;; See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message-verify-message-signature.html
|
||||
;;
|
||||
;; V1 and V2 use the SAME field sets (only the hash algorithm differs: SHA1 vs SHA256)
|
||||
;; The "Signature" field is NEVER part of the string-to-sign (it's the output, not input)
|
||||
;; "SigningCertURL" and "SignatureVersion" are metadata, not signed
|
||||
|
||||
;; Notification: Message, MessageId, Subject (if present), Timestamp, TopicArn, Type
|
||||
(def ^:private notification-fields
|
||||
["Message" "MessageId" "Subject" "Timestamp" "TopicArn" "Type"])
|
||||
|
||||
;; SubscriptionConfirmation: Message, MessageId, SubscribeURL, Timestamp, Token, TopicArn, Type
|
||||
(def ^:private subscription-fields
|
||||
["Message" "MessageId" "SubscribeURL" "Timestamp" "Token" "TopicArn" "Type"])
|
||||
|
||||
(defn- build-string-to-sign
|
||||
"Builds the string-to-sign for AWS SNS signature verification.
|
||||
V1 and V2 use the same field sets (only hash algorithm differs: SHA1 vs SHA256).
|
||||
See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message-verify-message-signature.html"
|
||||
[body]
|
||||
(let [msg-type (get body "Type")
|
||||
fields (if (= "SubscriptionConfirmation" msg-type)
|
||||
subscription-fields
|
||||
notification-fields)]
|
||||
(->> fields
|
||||
(filter #(contains? body %))
|
||||
(map #(str % "\n" (get body %) "\n"))
|
||||
(apply str))))
|
||||
|
||||
(defn- fetch-certificate
|
||||
"Fetches the X.509 certificate from the given URL.
|
||||
Returns an InputStream that must be closed by the caller.
|
||||
See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html"
|
||||
[cfg cert-url]
|
||||
(let [response (http/req cfg {:uri cert-url :method :get :timeout 10000}
|
||||
{:sync? true :response-type :input-stream})]
|
||||
(when-not (= 200 (:status response))
|
||||
(when-let [body (:body response)]
|
||||
(.close ^java.io.Closeable body))
|
||||
(l/wrn :hint "failed to fetch SNS signing certificate"
|
||||
:action "sns-cert-fetch-failed"
|
||||
:status (:status response)
|
||||
:cert-url cert-url)
|
||||
(ex/raise :type :internal :code :cert-fetch-failed))
|
||||
(:body response)))
|
||||
|
||||
(defn- verify-signature
|
||||
"Verifies the RSA signature of the message.
|
||||
See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html"
|
||||
[cfg body]
|
||||
(let [cert-url (get body "SigningCertURL")
|
||||
signature (get body "Signature")
|
||||
sig-version (get body "SignatureVersion")
|
||||
algorithm (case sig-version
|
||||
"1" "SHA1withRSA"
|
||||
"2" "SHA256withRSA"
|
||||
nil)]
|
||||
(when-not algorithm
|
||||
(throw (ex-info "Unsupported SNS signature version"
|
||||
{:type :validation :version sig-version})))
|
||||
(when (and cert-url signature)
|
||||
(try
|
||||
(let [string-sign (build-string-to-sign body)]
|
||||
(with-open [cert-stream (fetch-certificate cfg cert-url)]
|
||||
(let [cf (CertificateFactory/getInstance "X.509")
|
||||
cert (.generateCertificate cf cert-stream)
|
||||
sig (Signature/getInstance algorithm)]
|
||||
(.initVerify sig (.getPublicKey cert))
|
||||
(.update sig (.getBytes string-sign java.nio.charset.StandardCharsets/UTF_8))
|
||||
(.verify sig (.decode (Base64/getDecoder) signature)))))
|
||||
(catch clojure.lang.ExceptionInfo e
|
||||
(let [data (ex-data e)]
|
||||
(if (= :validation (:type data))
|
||||
(throw e)
|
||||
(do
|
||||
(l/wrn :hint "SNS signature verification exception"
|
||||
:action "sns-signature-verification-exception"
|
||||
:cause e)
|
||||
false))))
|
||||
(catch Exception e
|
||||
(l/wrn :hint "SNS signature verification exception"
|
||||
:action "sns-signature-verification-exception"
|
||||
:cause e)
|
||||
false)))))
|
||||
|
||||
(defn- verify-sns-message!
|
||||
"Verifies the AWS SNS message signature and URL validity.
|
||||
Throws if verification fails.
|
||||
See: https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html"
|
||||
[cfg body]
|
||||
(let [cert-url (get body "SigningCertURL")
|
||||
subscribe-url (get body "SubscribeURL")
|
||||
mtype (get body "Type")]
|
||||
|
||||
(when-not (valid-sns-url? cert-url)
|
||||
(l/wrn :hint "SNS certificate URL not from amazonaws.com"
|
||||
:action "sns-invalid-cert-url"
|
||||
:message-type mtype
|
||||
:signing-cert-url cert-url)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-signing-cert-url
|
||||
:hint "SigningCertURL must be from amazonaws.com"))
|
||||
|
||||
(when (and (= mtype "SubscriptionConfirmation")
|
||||
(not (valid-sns-url? subscribe-url)))
|
||||
(l/wrn :hint "SNS subscribe URL not from amazonaws.com"
|
||||
:action "sns-invalid-subscribe-url"
|
||||
:message-type mtype
|
||||
:subscribe-url subscribe-url)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-subscribe-url
|
||||
:hint "SubscribeURL must be from amazonaws.com"))
|
||||
|
||||
(when-not (verify-signature cfg body)
|
||||
(l/wrn :hint "SNS signature verification failed"
|
||||
:action "sns-signature-verification-failed"
|
||||
:message-type mtype
|
||||
:topic-arn (get body "TopicArn")
|
||||
:signing-cert-url cert-url)
|
||||
(ex/raise :type :authentication
|
||||
:code :invalid-signature
|
||||
:hint "SNS signature verification failed"))))
|
||||
|
||||
(defmethod ig/assert-key ::routes
|
||||
[_ params]
|
||||
(assert (http/client? (::http/client params)) "expect a valid http client")
|
||||
@@ -37,9 +182,9 @@
|
||||
(defmethod ig/init-key ::routes
|
||||
[_ cfg]
|
||||
(letfn [(handler [request]
|
||||
(let [data (-> request yreq/body slurp)]
|
||||
(handle-request cfg data)
|
||||
{::yres/status 200}))]
|
||||
(let [data (-> request yreq/body slurp)
|
||||
result (handle-request cfg data)]
|
||||
{::yres/status (or (:status result) 200)}))]
|
||||
["/sns" {:handler handler
|
||||
:allowed-methods #{:post}}]))
|
||||
|
||||
@@ -48,25 +193,52 @@
|
||||
(try
|
||||
(let [body (parse-json data)
|
||||
mtype (get body "Type")]
|
||||
|
||||
(when body
|
||||
(verify-sns-message! cfg body))
|
||||
|
||||
(cond
|
||||
(= mtype "SubscriptionConfirmation")
|
||||
(let [surl (get body "SubscribeURL")
|
||||
stopic (get body "TopicArn")]
|
||||
(l/info :action "subscription received" :topic stopic :url surl)
|
||||
(http/req cfg {:uri surl :method :post :timeout 10000} {:sync? true}))
|
||||
(http/req cfg {:uri surl :method :post :timeout 10000} {:sync? true})
|
||||
{:status 200})
|
||||
|
||||
(= mtype "Notification")
|
||||
(when-let [message (parse-json (get body "Message"))]
|
||||
(let [notification (parse-notification cfg message)]
|
||||
(process-report cfg notification)))
|
||||
(if-let [message (parse-json (get body "Message"))]
|
||||
(do
|
||||
(let [notification (parse-notification cfg message)]
|
||||
(process-report cfg notification))
|
||||
{:status 200})
|
||||
(do
|
||||
(l/wrn :hint "notification with missing or unparseable Message field"
|
||||
:action "sns-missing-message")
|
||||
{:status 400}))
|
||||
|
||||
:else
|
||||
(l/warn :hint "unexpected data received"
|
||||
:report (pr-str body))))
|
||||
(do
|
||||
(l/warn :hint "unexpected data received"
|
||||
:report (pr-str body))
|
||||
{:status 400})))
|
||||
|
||||
(catch clojure.lang.ExceptionInfo e
|
||||
(let [data (ex-data e)]
|
||||
(if (#{:validation :authentication} (:type data))
|
||||
(do
|
||||
(l/wrn :hint "SNS message validation failed"
|
||||
:action "sns-validation-failed"
|
||||
:code (:code data))
|
||||
{:status 400})
|
||||
(do
|
||||
(l/error :hint "unexpected exception on awsns"
|
||||
:cause e)
|
||||
{:status 500}))))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/error :hint "unexpected exception on awsns"
|
||||
:cause cause))))
|
||||
:cause cause)
|
||||
{:status 500})))
|
||||
|
||||
(defn- parse-bounce
|
||||
[data]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.client
|
||||
"Http client abstraction layer.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.debug
|
||||
(:refer-clojure :exclude [error-handler])
|
||||
@@ -13,9 +13,6 @@
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.changes :as cfc]
|
||||
[app.common.files.repair :as cfr]
|
||||
[app.common.files.validate :as cfv]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.time :as ct]
|
||||
@@ -31,7 +28,6 @@
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.setup :as-alias setup]
|
||||
[app.setup.clock :as clock]
|
||||
[app.srepl.helpers :as h]
|
||||
[app.srepl.main :as srepl]
|
||||
[app.storage :as-alias sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
@@ -134,7 +130,7 @@
|
||||
:hint "invalid button"))
|
||||
|
||||
(ex/raise :type :not-found
|
||||
:code :empty-data
|
||||
:code :enpty-data
|
||||
:hint "empty response"))))
|
||||
|
||||
(defn- is-file-exists?
|
||||
@@ -488,89 +484,6 @@
|
||||
{::yres/status 302
|
||||
::yres/headers {"location" "/dbg"}}))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; VALIDATE / REPAIR
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- validate-file
|
||||
[cfg {:keys [params] :as request}]
|
||||
(let [file-id (some-> params :file-id parse-uuid)]
|
||||
|
||||
(when-not file-id
|
||||
(ex/raise :type :validation
|
||||
:code :missing-arguments))
|
||||
|
||||
(db/tx-run! (assoc cfg ::db/rollback true)
|
||||
(fn [cfg]
|
||||
(let [file (bfc/get-file cfg file-id)
|
||||
libs (bfc/get-resolved-file-libraries cfg file-id)]
|
||||
(if file
|
||||
(let [errors (cfv/validate-file file libs)]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/plain"}
|
||||
::yres/body (if (empty? errors)
|
||||
"NO VALIDATION ERRORS FOUND"
|
||||
(pp/pprint-str errors))})
|
||||
(ex/raise :type :not-found
|
||||
:code :empty-data
|
||||
:hint "empty response")))))))
|
||||
|
||||
(defn- repair-file
|
||||
[cfg {:keys [params] :as request}]
|
||||
(let [file-id (some-> params :file-id parse-uuid)
|
||||
skip-snapshot? (contains? params :skip-snapshot)
|
||||
profile-id (:app.http.session/profile-id request)]
|
||||
|
||||
(when-not file-id
|
||||
(ex/raise :type :validation
|
||||
:code :missing-arguments))
|
||||
|
||||
(let [output (StringBuilder.)
|
||||
|
||||
repair-file
|
||||
(fn [file libs _]
|
||||
(let [errors (cfv/validate-file file libs)]
|
||||
(.append output (if (empty? errors)
|
||||
"NO VALIDATION ERRORS FOUND\n"
|
||||
(str "VALIDATION ERRORS FOUND:\n"
|
||||
(pp/pprint-str errors) "\n")))
|
||||
(if (empty? errors)
|
||||
file
|
||||
(let [changes (cfr/repair-file file libs errors)]
|
||||
(-> file
|
||||
(update :revn inc)
|
||||
(update :data cfc/process-changes changes))))))]
|
||||
|
||||
(add-watch l/log-record ::repair-watcher
|
||||
(fn [_ _ _ record]
|
||||
(when (= "app.common.files.repair" (::l/logger record))
|
||||
(let [props (::l/props record)
|
||||
hint (get props :hint "")
|
||||
args (dissoc props :hint)
|
||||
message (str hint " "
|
||||
(when-not (empty? args)
|
||||
args)
|
||||
"\n")]
|
||||
(.append output message)))))
|
||||
(try
|
||||
(db/tx-run! cfg
|
||||
h/process-file!
|
||||
file-id
|
||||
repair-file
|
||||
{::h/with-libraries? true
|
||||
::h/validate? false
|
||||
::h/profile-id profile-id
|
||||
::h/snapshot-label (when-not skip-snapshot? "repair")})
|
||||
|
||||
(.append output "\nREPAIR FINISHED")
|
||||
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/plain"}
|
||||
::yres/body (.toString output)}
|
||||
|
||||
(finally
|
||||
(remove-watch l/log-record ::repair-watcher))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; OTHER SMALL VIEWS/HANDLERS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
@@ -665,7 +578,5 @@
|
||||
{:handler (partial handle-team-features cfg)}]
|
||||
["/file-export" {:handler (partial export-handler cfg)}]
|
||||
["/file-import" {:handler (partial import-handler cfg)}]
|
||||
["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]
|
||||
["/file-validate" {:handler (partial validate-file cfg)}]
|
||||
["/file-repair" {:handler (partial repair-file cfg)}]]]])
|
||||
["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.errors
|
||||
"A errors handling for the http server."
|
||||
@@ -34,12 +34,6 @@
|
||||
(assoc :request/auth-data (dissoc auth :token))
|
||||
(assoc :frontend/version (or (yreq/get-header request "x-frontend-version") "unknown")))))
|
||||
|
||||
(defn- strip-internal-fields
|
||||
"Remove fields that leak internal implementation details from error
|
||||
response data. Full context is preserved in server-side logs."
|
||||
[data]
|
||||
(dissoc data :state :path :context))
|
||||
|
||||
(defmulti handle-error
|
||||
(fn [cause _ _]
|
||||
(-> cause ex-data :type)))
|
||||
@@ -71,12 +65,9 @@
|
||||
|
||||
(defmethod handle-error :rate-limit
|
||||
[err _ _]
|
||||
(let [data (ex-data err)]
|
||||
(let [headers (-> err ex-data ::http/headers)]
|
||||
{::yres/status 429
|
||||
::yres/body {:type :rate-limit
|
||||
:code (:code data)
|
||||
:hint (:hint data)
|
||||
:ttl (:ttl data)}}))
|
||||
::yres/headers headers}))
|
||||
|
||||
(defmethod handle-error :concurrency-limit
|
||||
[err _ _]
|
||||
@@ -145,7 +136,6 @@
|
||||
(l/error :hint "assertion error" :cause cause)
|
||||
{::yres/status 500
|
||||
::yres/body (-> data
|
||||
(strip-internal-fields)
|
||||
(assoc :type :server-error)
|
||||
(assoc :code :assertion))})))))
|
||||
|
||||
@@ -171,9 +161,9 @@
|
||||
(l/error :hint "internal error" :cause cause)
|
||||
{::yres/status 500
|
||||
::yres/body (-> data
|
||||
(strip-internal-fields)
|
||||
(assoc :type :server-error)
|
||||
(update :code #(or % :unhandled)))})))
|
||||
(update :code #(or % :unhandled))
|
||||
(assoc :hint (ex-message error)))})))
|
||||
|
||||
(defmethod handle-error :default
|
||||
[error request parent-cause]
|
||||
@@ -188,20 +178,6 @@
|
||||
(handle-exception (:handling edata) request error)
|
||||
(handle-exception error request parent-cause))))
|
||||
|
||||
(defn- pgsql-state->message
|
||||
"Map PostgreSQL SQLSTATE codes to safe, client-facing messages.
|
||||
Returns a user-friendly string that conveys the nature of the error
|
||||
without exposing table names, constraint names, or other internals."
|
||||
[state]
|
||||
(case state
|
||||
"23505" "A conflicting entry already exists"
|
||||
"23503" "The referenced item does not exist"
|
||||
"23502" "A required field is missing"
|
||||
"23514" "The value violates a data integrity constraint"
|
||||
"57014" "The operation took too long and was cancelled"
|
||||
"25P03" "The transaction was idle too long and was cancelled"
|
||||
"A database error occurred"))
|
||||
|
||||
(defmethod handle-exception org.postgresql.util.PSQLException
|
||||
[error request parent-cause]
|
||||
(let [state (.getSQLState ^java.sql.SQLException error)
|
||||
@@ -214,19 +190,20 @@
|
||||
{::yres/status 504
|
||||
::yres/body {:type :server-error
|
||||
:code :statement-timeout
|
||||
:hint (pgsql-state->message state)}}
|
||||
:hint (ex-message error)}}
|
||||
|
||||
(= state "25P03")
|
||||
{::yres/status 504
|
||||
::yres/body {:type :server-error
|
||||
:code :idle-in-transaction-timeout
|
||||
:hint (pgsql-state->message state)}}
|
||||
:hint (ex-message error)}}
|
||||
|
||||
:else
|
||||
{::yres/status 500
|
||||
::yres/body {:type :server-error
|
||||
:code :database-error
|
||||
:hint (pgsql-state->message state)}}))))
|
||||
:code :unexpected
|
||||
:hint (ex-message error)
|
||||
:state state}}))))
|
||||
|
||||
(defmethod handle-exception :default
|
||||
[error request parent-cause]
|
||||
@@ -239,16 +216,17 @@
|
||||
(l/error :hint "unexpected error" :cause cause)
|
||||
{::yres/status 500
|
||||
::yres/body {:type :server-error
|
||||
:code :unexpected}})
|
||||
:code :unexpected
|
||||
:hint (ex-message error)}})
|
||||
|
||||
:else
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "unhandled error" :cause cause)
|
||||
{::yres/status 500
|
||||
::yres/body (-> edata
|
||||
(strip-internal-fields)
|
||||
(assoc :type :server-error)
|
||||
(update :code #(or % :unhandled)))}))))
|
||||
(update :code #(or % :unhandled))
|
||||
(assoc :hint (ex-message error)))}))))
|
||||
|
||||
(defmethod handle-exception java.io.IOException
|
||||
[cause request _]
|
||||
@@ -256,7 +234,9 @@
|
||||
(l/wrn :hint "io exception" :cause cause)
|
||||
{::yres/status 500
|
||||
::yres/body {:type :server-error
|
||||
:code :io-exception}}))
|
||||
:code :io-exception
|
||||
:hint (ex-message cause)
|
||||
:path (:path request)}}))
|
||||
|
||||
(defmethod handle-exception java.util.concurrent.CompletionException
|
||||
[cause request _]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.management
|
||||
"Internal mangement HTTP API"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.middleware
|
||||
(:require
|
||||
@@ -24,8 +24,7 @@
|
||||
(:import
|
||||
io.undertow.server.RequestTooBigException
|
||||
java.io.InputStream
|
||||
java.io.OutputStream
|
||||
java.security.MessageDigest))
|
||||
java.io.OutputStream))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
@@ -83,18 +82,18 @@
|
||||
(instance? IllegalArgumentException cause)
|
||||
(ex/raise :type :validation
|
||||
:code :malformed-json
|
||||
:hint "invalid JSON in request body"
|
||||
:hint (ex-message cause)
|
||||
:cause cause)
|
||||
|
||||
(instance? RequestTooBigException cause)
|
||||
(ex/raise :type :validation
|
||||
:code :request-body-too-large
|
||||
:hint "request body exceeds size limit")
|
||||
:hint (ex-message cause))
|
||||
|
||||
(instance? java.io.EOFException cause)
|
||||
(ex/raise :type :validation
|
||||
:code :malformed-json
|
||||
:hint "unexpected end of request body"
|
||||
:hint (ex-message cause)
|
||||
:cause cause)
|
||||
|
||||
(instance? RuntimeException cause)
|
||||
@@ -330,11 +329,6 @@
|
||||
{:name ::auth
|
||||
:compile (constantly wrap-auth)})
|
||||
|
||||
(defn- constant-time-eq?
|
||||
"Compare strings in constant time to prevent timing attacks."
|
||||
[^String a ^String b]
|
||||
(MessageDigest/isEqual (.getBytes a "UTF-8") (.getBytes b "UTF-8")))
|
||||
|
||||
(defn- wrap-shared-key-auth
|
||||
[handler keys]
|
||||
(if (seq keys)
|
||||
@@ -344,7 +338,7 @@
|
||||
(let [key-id (-> key-id str/lower keyword)]
|
||||
(if (and (string? key)
|
||||
(contains? keys key-id)
|
||||
(constant-time-eq? key (get keys key-id)))
|
||||
(= key (get keys key-id)))
|
||||
(-> request
|
||||
(assoc ::http/auth-key-id key-id)
|
||||
(handler))
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.security
|
||||
"Additional security layer middlewares"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.session
|
||||
(:refer-clojure :exclude [read])
|
||||
@@ -204,7 +204,7 @@
|
||||
[{:keys [::manager]}]
|
||||
(assert (manager? manager) "expected valid session manager")
|
||||
(fn [request response]
|
||||
(some->> (get request ::session) :id (delete-session manager))
|
||||
(some->> (get request ::id) (delete-session manager))
|
||||
(clear-session-cookie response)))
|
||||
|
||||
(defn decode-token
|
||||
@@ -226,14 +226,6 @@
|
||||
(-> (db/exec-one! cfg [sql (:profile-id session) (:id session)])
|
||||
(db/get-update-count))))
|
||||
|
||||
(defn invalidate-all
|
||||
"Delete all sessions for a given profile. Used when a profile is deleted
|
||||
to ensure immediate access revocation across all devices."
|
||||
[cfg profile-id]
|
||||
(let [sql "delete from http_session_v2 where profile_id = ?"]
|
||||
(-> (db/exec-one! cfg [sql profile-id])
|
||||
(db/get-update-count))))
|
||||
|
||||
(def ^:private sql:clear-organization-sso-sessions
|
||||
(str "UPDATE http_session_v2 "
|
||||
"SET props = props #- ARRAY['~:sso', ?]::text[] "
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.sse
|
||||
"SSE (server sent events) helpers"
|
||||
|
||||
@@ -2,12 +2,11 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.http.websocket
|
||||
"A penpot notification service for file cooperative edition."
|
||||
(:require
|
||||
[app.binfile.common :as bfc]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
@@ -18,8 +17,6 @@
|
||||
[app.http.session :as session]
|
||||
[app.metrics :as mtx]
|
||||
[app.msgbus :as mbus]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.util.websocket :as ws]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec.csp :as sp]
|
||||
@@ -134,9 +131,8 @@
|
||||
(mbus/pub! msgbus :topic topic :message msg))))
|
||||
|
||||
(defmethod handle-message :subscribe-team
|
||||
[{:keys [::mbus/msgbus ::db/pool]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [team-id] :as params}]
|
||||
[{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id]} {:keys [team-id] :as params}]
|
||||
(l/trace :fn "handle-message" :event "subscribe-team" :team-id team-id :conn-id id)
|
||||
(teams/check-read-permissions! pool profile-id team-id)
|
||||
(let [prev-subs (get @state ::team-subscription)
|
||||
channel (sp/chan :buf (sp/dropping-buffer 64)
|
||||
:xf (remove #(= (:session-id %) session-id)))]
|
||||
@@ -154,10 +150,8 @@
|
||||
|
||||
|
||||
(defmethod handle-message :subscribe-file
|
||||
[{:keys [::mbus/msgbus ::db/pool]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [file-id] :as params}]
|
||||
[{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id ::profile-id]} {:keys [file-id] :as params}]
|
||||
(l/trace :fn "handle-message" :event "subscribe-file" :file-id file-id :conn-id id)
|
||||
(bfc/check-file-exists pool file-id)
|
||||
(files/check-read-permissions! pool profile-id file-id)
|
||||
(let [psub (::file-subscription @state)
|
||||
fch (sp/chan :buf (sp/dropping-buffer 64)
|
||||
:xf (remove #(= (:session-id %) session-id)))]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.audit
|
||||
"Services related to the user activity (audit log)."
|
||||
@@ -36,16 +36,6 @@
|
||||
(def ^:private filter-auth-events
|
||||
#{"login-with-oidc" "login-with-password" "register-profile" "update-profile"})
|
||||
|
||||
(def ^:private organization-sso-failure-reasons
|
||||
#{"access-denied"
|
||||
"provider-unavailable"
|
||||
"invalid-configuration"
|
||||
"provider-error"
|
||||
"token-exchange-failed"
|
||||
"user-info-failed"
|
||||
"incomplete-user-info"
|
||||
"unexpected-error"})
|
||||
|
||||
(def ^:private safe-backend-context-keys
|
||||
#{:version
|
||||
:initiator
|
||||
@@ -307,14 +297,6 @@
|
||||
(defn filter-telemetry-props
|
||||
[{:keys [source name props type] :as params}]
|
||||
(cond
|
||||
(and (= source "backend")
|
||||
(= name "organization-sso-auth-failed"))
|
||||
(let [props' (into {} xf:filter-telemetry-props props)
|
||||
props' (cond-> props'
|
||||
(contains? organization-sso-failure-reasons (:failure-reason props))
|
||||
(assoc :failure-reason (:failure-reason props)))]
|
||||
(assoc params :props props'))
|
||||
|
||||
(or (and (= source "frontend")
|
||||
(= type "identify"))
|
||||
(and (= source "backend")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.audit.archive-task
|
||||
(:require
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.audit.gc-task
|
||||
(:require
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.database
|
||||
"A specific logger impl that persists errors on the database."
|
||||
|
||||
@@ -2,12 +2,11 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.mattermost
|
||||
"A mattermost integration for error reporting."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
@@ -26,7 +25,7 @@
|
||||
(defn- send-mattermost-notification!
|
||||
[cfg {:keys [id] :as report}]
|
||||
(let [type (get report :type)
|
||||
text (str "#" type " | " (d/escape-markdown (get report :hint)) "\n"
|
||||
text (str "#" type " | " (get report :hint) "\n"
|
||||
(when id
|
||||
(str (u/join (cf/get :public-uri) "/dbg/error/" id) " "))
|
||||
|
||||
@@ -39,7 +38,7 @@
|
||||
"- tenant: #" (:tenant report) "\n"
|
||||
"- origin: #" (:origin report) "\n"
|
||||
(when-let [href (get report :href)]
|
||||
(str "- href: `" (d/escape-markdown href) "`\n"))
|
||||
(str "- href: `" href "`\n"))
|
||||
(when-let [version (get report :frontend-version)]
|
||||
(str "- frontend-version: `" version "`\n"))
|
||||
(when-let [version (get report :backend-version)]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.loggers.webhooks
|
||||
"A mattermost integration for error reporting."
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.main
|
||||
(:require
|
||||
@@ -37,7 +37,6 @@
|
||||
[app.storage.fs :as-alias sto.fs]
|
||||
[app.storage.gc-deleted :as-alias sto.gc-deleted]
|
||||
[app.storage.gc-touched :as-alias sto.gc-touched]
|
||||
[app.storage.pending-gc :as-alias sto.pending-gc]
|
||||
[app.storage.s3 :as-alias sto.s3]
|
||||
[app.system :as sys]
|
||||
[app.util.cron]
|
||||
@@ -200,10 +199,6 @@
|
||||
::sto.gc-touched/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
::sto.pending-gc/handler
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
::http.client/client
|
||||
{}
|
||||
|
||||
@@ -391,7 +386,6 @@
|
||||
:upload-session-gc (ig/ref :app.tasks.upload-session-gc/handler)
|
||||
:storage-gc-deleted (ig/ref ::sto.gc-deleted/handler)
|
||||
:storage-gc-touched (ig/ref ::sto.gc-touched/handler)
|
||||
:storage-pending-gc (ig/ref ::sto.pending-gc/handler)
|
||||
:session-gc (ig/ref ::session.tasks/gc)
|
||||
:audit-log-archive (ig/ref :app.loggers.audit.archive-task/handler)
|
||||
:audit-log-gc (ig/ref :app.loggers.audit.gc-task/handler)
|
||||
@@ -551,9 +545,6 @@
|
||||
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
|
||||
:task :storage-gc-touched}
|
||||
|
||||
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
|
||||
:task :storage-pending-gc}
|
||||
|
||||
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
|
||||
:task :tasks-gc}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.media
|
||||
"Media & Font postprocessing.
|
||||
|
||||
@@ -2,27 +2,35 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.media.local
|
||||
"Local media processing via ImageMagick and FontForge shell commands."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.media :as cm]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.config :as cf]
|
||||
[app.media.svg :as svg]
|
||||
[app.media.validation :as validation]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.shell :as shell]
|
||||
[buddy.core.bytes :as bb]
|
||||
[buddy.core.codecs :as bc]
|
||||
[clojure.string]
|
||||
[clojure.xml :as xml]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
[datoteka.io :as io]))
|
||||
[datoteka.io :as io])
|
||||
(:import
|
||||
clojure.lang.XMLHandler
|
||||
java.io.InputStream
|
||||
javax.xml.parsers.SAXParserFactory
|
||||
javax.xml.XMLConstants
|
||||
org.apache.commons.io.IOUtils))
|
||||
|
||||
(defmulti process (fn [_system params] (:cmd params)))
|
||||
|
||||
@@ -32,6 +40,30 @@
|
||||
:code :not-implemented
|
||||
:hint (str/fmt "No impl found for local process cmd: %s" cmd)))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; SVG PARSING
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- secure-parser-factory
|
||||
[^InputStream input ^XMLHandler handler]
|
||||
(.. (doto (SAXParserFactory/newInstance)
|
||||
(.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
|
||||
(.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
|
||||
(newSAXParser)
|
||||
(parse input handler)))
|
||||
|
||||
(defn- strip-doctype
|
||||
[data]
|
||||
(cond-> data
|
||||
(str/includes? data "<!DOCTYPE")
|
||||
(str/replace #"<\!DOCTYPE[^>]*>" "")))
|
||||
|
||||
(defn parse-svg
|
||||
[text]
|
||||
(let [text (strip-doctype text)]
|
||||
(dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
|
||||
(xml/parse istream secure-parser-factory))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; IMAGE THUMBNAILS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
@@ -135,6 +167,34 @@
|
||||
"-extent" (str width "x" height)
|
||||
"-quality" (str quality)]))))
|
||||
|
||||
(defn get-basic-info-from-svg
|
||||
[{:keys [tag attrs] :as data}]
|
||||
(when (not= tag :svg)
|
||||
(ex/raise :type :validation
|
||||
:code :unable-to-parse-svg
|
||||
:hint "uploaded svg has invalid content"))
|
||||
(reduce (fn [default f]
|
||||
(if-let [res (f attrs)]
|
||||
(reduced res)
|
||||
default))
|
||||
{:width 100 :height 100}
|
||||
[(fn parse-width-and-height
|
||||
[{:keys [width height]}]
|
||||
(when (and (string? width)
|
||||
(string? height))
|
||||
(let [width (d/parse-double width)
|
||||
height (d/parse-double height)]
|
||||
(when (and width height)
|
||||
{:width (int width)
|
||||
:height (int height)}))))
|
||||
(fn parse-viewbox
|
||||
[{:keys [viewBox]}]
|
||||
(let [[x y width height] (->> (str/split viewBox #"\s+" 4)
|
||||
(map d/parse-double))]
|
||||
(when (and x y width height)
|
||||
{:width (int width)
|
||||
:height (int height)})))]))
|
||||
|
||||
(defn- get-dimensions-with-orientation [system ^String path]
|
||||
;; Image magick doesn't give info about exif rotation so we use the identify command
|
||||
;; If we are processing an animated gif we use the first frame with -scene 0
|
||||
@@ -157,7 +217,7 @@
|
||||
[system {:keys [input] :as params}]
|
||||
(let [{:keys [path mtype] :as input} (validation/check-input input)]
|
||||
(if (= mtype "image/svg+xml")
|
||||
(let [info (some-> path slurp svg/parse-svg svg/get-basic-info-from-svg)]
|
||||
(let [info (some-> path slurp parse-svg get-basic-info-from-svg)]
|
||||
(when-not info
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-svg-file
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.media.remote
|
||||
"Remote media processing via the media-processor HTTP service."
|
||||
@@ -13,7 +13,7 @@
|
||||
[app.common.uri :as uri]
|
||||
[app.config :as cf]
|
||||
[app.http.client :as http]
|
||||
[app.media.svg :as svg]
|
||||
[app.media.local :as local]
|
||||
[app.media.validation :as validation]
|
||||
[app.setup :as-alias setup]
|
||||
[app.storage.tmp :as tmp]
|
||||
@@ -182,7 +182,7 @@
|
||||
(let [{:keys [path mtype]} (validation/check-input input)]
|
||||
(if (= mtype "image/svg+xml")
|
||||
;; SVG: parse locally (Sharp doesn't support SVG)
|
||||
(let [info (some-> path slurp svg/parse-svg svg/get-basic-info-from-svg)]
|
||||
(let [info (some-> path slurp local/parse-svg local/get-basic-info-from-svg)]
|
||||
(when-not info
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-svg-file
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.media.sanitize
|
||||
"Image EOF truncation helpers — strips trailing data after image EOF
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.media.svg
|
||||
"SVG parsing, sanitization, and info extraction.
|
||||
Centralizes all SVG-related security concerns."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[clojure.xml :as xml]
|
||||
[cuerdas.core :as str])
|
||||
(:import
|
||||
clojure.lang.XMLHandler
|
||||
java.io.InputStream
|
||||
javax.xml.parsers.SAXParserFactory
|
||||
javax.xml.XMLConstants
|
||||
org.apache.commons.io.IOUtils))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; SVG PARSING
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- secure-parser-factory
|
||||
[^InputStream input ^XMLHandler handler]
|
||||
(.. (doto (SAXParserFactory/newInstance)
|
||||
(.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
|
||||
(.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
|
||||
(newSAXParser)
|
||||
(parse input handler)))
|
||||
|
||||
(defn- strip-doctype
|
||||
[data]
|
||||
(cond-> data
|
||||
(str/includes? data "<!DOCTYPE")
|
||||
(str/replace #"<\!DOCTYPE[^>]*>" "")))
|
||||
|
||||
(defn parse-svg
|
||||
[text]
|
||||
(let [text (strip-doctype text)]
|
||||
(dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
|
||||
(xml/parse istream secure-parser-factory))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; SVG SANITIZATION
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(def ^:private dangerous-attrs-pattern #"(?i)^on\w+$")
|
||||
(def ^:private javascript-href-pattern #"(?i)^javascript:")
|
||||
|
||||
(defn- sanitize-svg-element
|
||||
"Recursively sanitize an SVG element by removing dangerous tags and attributes."
|
||||
[{:keys [tag attrs content] :as element}]
|
||||
(when (and (map? element) tag)
|
||||
(let [dangerous-tags #{:script :foreignObject :set :animate :animateTransform :animateColor :animateMotion}]
|
||||
(when-not (contains? dangerous-tags tag)
|
||||
(let [clean-attrs (->> attrs
|
||||
(remove (fn [[k v]]
|
||||
(or (re-matches dangerous-attrs-pattern (name k))
|
||||
(and (#{:href :xlink:href} k)
|
||||
(string? v)
|
||||
(re-find javascript-href-pattern (str/trim v))))))
|
||||
(into {}))
|
||||
clean-content (when content
|
||||
(->> content
|
||||
(filter #(or (string? %) (map? %)))
|
||||
(map (fn [child]
|
||||
(if (map? child)
|
||||
(sanitize-svg-element child)
|
||||
child)))
|
||||
(filter some?)
|
||||
vec))]
|
||||
(cond-> {:tag tag :attrs clean-attrs}
|
||||
(seq clean-content) (assoc :content clean-content)))))))
|
||||
|
||||
(defn sanitize-svg
|
||||
"Sanitize SVG content by removing dangerous elements and attributes.
|
||||
Removes <script> tags, <foreignObject> elements, event handlers (on*),
|
||||
and javascript: URLs from href attributes."
|
||||
[svg-text]
|
||||
(try
|
||||
(let [parsed (parse-svg svg-text)
|
||||
sanitized (sanitize-svg-element parsed)]
|
||||
(if sanitized
|
||||
(with-out-str (xml/emit sanitized))
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-svg-file
|
||||
:hint "SVG sanitization produced no output")))
|
||||
(catch Exception e
|
||||
(l/warn :hint "SVG sanitization failed, rejecting upload" :cause e)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-svg-file
|
||||
:hint "SVG parsing failed during sanitization"
|
||||
:cause e))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; SVG INFO EXTRACTION
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn get-basic-info-from-svg
|
||||
[{:keys [tag attrs] :as data}]
|
||||
(when (not= tag :svg)
|
||||
(ex/raise :type :validation
|
||||
:code :unable-to-parse-svg
|
||||
:hint "uploaded svg has invalid content"))
|
||||
(reduce (fn [default f]
|
||||
(if-let [res (f attrs)]
|
||||
(reduced res)
|
||||
default))
|
||||
{:width 100 :height 100}
|
||||
[(fn parse-width-and-height
|
||||
[{:keys [width height]}]
|
||||
(when (and (string? width)
|
||||
(string? height))
|
||||
(let [width (d/parse-double width)
|
||||
height (d/parse-double height)]
|
||||
(when (and width height)
|
||||
{:width (int width)
|
||||
:height (int height)}))))
|
||||
(fn parse-viewbox
|
||||
[{:keys [viewBox]}]
|
||||
(let [[x y width height] (->> (str/split viewBox #"\s+" 4)
|
||||
(map d/parse-double))]
|
||||
(when (and x y width height)
|
||||
{:width (int width)
|
||||
:height (int height)})))]))
|
||||
@@ -2,7 +2,7 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.media.validation
|
||||
"Schemas and validation functions for media uploads.
|
||||
|
||||
Loaded 100 of 1808 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user