mirror of
https://github.com/penpot/penpot.git
synced 2026-09-08 11:54:36 -04:00
Compare commits
103
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ccf7891479 | ||
|
|
deb0583029 | ||
|
|
a5b19118b9 | ||
|
|
0f66caf542 | ||
|
|
5b97fb9408 | ||
|
|
269aa36f82 | ||
|
|
1f12561427 | ||
|
|
c52778d6f6 | ||
|
|
9462543fb5 | ||
|
|
5c10ea5bd6 | ||
|
|
cd98a88c4d | ||
|
|
1dfa2cd9f2 | ||
|
|
f0680cf5f8 | ||
|
|
7ebd7cc0d5 | ||
|
|
2100ed29ea | ||
|
|
7e1139b906 | ||
|
|
960209f1fa | ||
|
|
9289e676e1 | ||
|
|
a8e0b3c1f8 | ||
|
|
c8691d0cde | ||
|
|
00e0492bb5 | ||
|
|
d82038a570 | ||
|
|
5452eb5c8a | ||
|
|
0533be100d | ||
|
|
df48c834e3 | ||
|
|
86aebd3429 | ||
|
|
7f46c8c80e | ||
|
|
fdcc4666e3 | ||
|
|
064f200774 | ||
|
|
214dbd0c8b | ||
|
|
f2f352d06c | ||
|
|
034707a9d7 | ||
|
|
eb8d3a2fef | ||
|
|
fb22c1547c | ||
|
|
7b26949c76 | ||
|
|
37f1d3f0fd | ||
|
|
a4becb5d1f | ||
|
|
a19b3c8d62 | ||
|
|
80dea409c6 | ||
|
|
7c762d8a98 | ||
|
|
15dff4a9e1 | ||
|
|
c6a32a2f5a | ||
|
|
f633d82f51 | ||
|
|
f22abc9861 | ||
|
|
c5897bc50a | ||
|
|
cb80ea98ba | ||
|
|
b2fb63988f | ||
|
|
70b443a716 | ||
|
|
88d715938f | ||
|
|
7620386215 | ||
|
|
b6b1a47a7b | ||
|
|
b92112da54 | ||
|
|
03e6f119e5 | ||
|
|
b9ddfc1596 | ||
|
|
b46ed37141 | ||
|
|
d1a54134bb | ||
|
|
99378dc02d | ||
|
|
c7b3a0849f | ||
|
|
b458dc764e | ||
|
|
f5aad7b1ae | ||
|
|
41ec134ea2 | ||
|
|
9051b87f0e | ||
|
|
a1079cf788 | ||
|
|
5c4b35e350 | ||
|
|
23869f3b1b | ||
|
|
810f8ef671 | ||
|
|
6d9f411fab | ||
|
|
29376d033b | ||
|
|
45f0153e8f | ||
|
|
326d83e780 | ||
|
|
15195b3bbb | ||
|
|
3df039abc3 | ||
|
|
92c2079ae2 | ||
|
|
5b3a1d9360 | ||
|
|
73d3d63616 | ||
|
|
89a17df403 | ||
|
|
736b99076f | ||
|
|
66b4a99ac3 | ||
|
|
ac5c88be66 | ||
|
|
36cf4f39e1 | ||
|
|
fc207a1768 | ||
|
|
93ac6d8338 | ||
|
|
806f947360 | ||
|
|
bfb5cae9f4 | ||
|
|
292390f0eb | ||
|
|
2ed0981e8a | ||
|
|
169e6fcbca | ||
|
|
995a5460e5 | ||
|
|
4c9aeab65e | ||
|
|
c01306734a | ||
|
|
10e40e62a2 | ||
|
|
f06d1df5be | ||
|
|
2ce202c7d8 | ||
|
|
0a14a1f3db | ||
|
|
eb0c51ed74 | ||
|
|
18ebf5a92d | ||
|
|
cdfdbf9adc | ||
|
|
38004e6bb2 | ||
|
|
1baa8bb29f | ||
|
|
11224940ab | ||
|
|
43bbf756b0 | ||
|
|
6bedd5ac76 | ||
|
|
6f35348c7c |
No files matched your search
@@ -41,7 +41,7 @@ jobs:
|
||||
outputs:
|
||||
gh_ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
bundle_version: ${{ steps.vars.outputs.bundle_version }}
|
||||
build_key: ${{ steps.vars.outputs.build_key }}
|
||||
sha: ${{ steps.vars.outputs.sha }}
|
||||
exists: ${{ steps.check.outputs.exists }}
|
||||
|
||||
steps:
|
||||
@@ -59,6 +59,7 @@ jobs:
|
||||
run: |
|
||||
GH_REF="${{ inputs.gh_ref || github.ref_name }}"
|
||||
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
|
||||
echo "sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
BUNDLE_VERSION=$(aws s3api head-object \
|
||||
--bucket ${{ secrets.S3_BUCKET }} \
|
||||
@@ -67,15 +68,10 @@ jobs:
|
||||
--output text)
|
||||
echo "bundle_version=$BUNDLE_VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
# Image content = bundle + docker build context, so the build key
|
||||
# combines both.
|
||||
CTX_HASH=$(git rev-parse "HEAD:docker/images" | cut -c1-12)
|
||||
echo "build_key=${BUNDLE_VERSION}-${CTX_HASH}" >> $GITHUB_OUTPUT
|
||||
|
||||
# The image set is a single block, so a single set-level check is
|
||||
# enough: `promote` drops a marker object in S3 only after every
|
||||
# image was built AND every branch tag was moved. Marker present
|
||||
# means there is nothing at all to do for this build key.
|
||||
# means there is nothing at all to do for this commit.
|
||||
- name: Check if this image set is already built
|
||||
id: check
|
||||
env:
|
||||
@@ -85,13 +81,13 @@ jobs:
|
||||
run: |
|
||||
if aws s3api head-object \
|
||||
--bucket ${{ secrets.S3_BUCKET }} \
|
||||
--key "markers/images-${{ steps.vars.outputs.build_key }}" \
|
||||
--key "markers/images-sha-${{ steps.vars.outputs.sha }}" \
|
||||
> /dev/null 2>&1; then
|
||||
echo "exists=true" >> $GITHUB_OUTPUT
|
||||
{
|
||||
echo "### ⏭️ Image set build skipped"
|
||||
echo ""
|
||||
echo "The whole set was already built and promoted for \`${{ steps.vars.outputs.build_key }}\`."
|
||||
echo "The whole set was already built and promoted for \`sha-${{ steps.vars.outputs.sha }}\`."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "exists=false" >> $GITHUB_OUTPUT
|
||||
@@ -101,7 +97,7 @@ jobs:
|
||||
# prune stale bundles while at it.
|
||||
mkdir -p "$BUNDLE_CACHE"
|
||||
find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true
|
||||
ZIP="$BUNDLE_CACHE/penpot-${{ steps.vars.outputs.build_key }}.zip"
|
||||
ZIP="$BUNDLE_CACHE/penpot-${{ steps.vars.outputs.bundle_version }}.zip"
|
||||
if [ ! -f "$ZIP" ]; then
|
||||
aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip" "$ZIP.$$.tmp"
|
||||
mv "$ZIP.$$.tmp" "$ZIP"
|
||||
@@ -173,7 +169,7 @@ jobs:
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
|
||||
run: |
|
||||
ZIP="$BUNDLE_CACHE/penpot-${{ needs.prepare.outputs.build_key }}.zip"
|
||||
ZIP="$BUNDLE_CACHE/penpot-${{ needs.prepare.outputs.bundle_version }}.zip"
|
||||
if [ ! -f "$ZIP" ]; then
|
||||
echo "Bundle not found in host cache; falling back to S3."
|
||||
mkdir -p "$BUNDLE_CACHE"
|
||||
@@ -213,7 +209,7 @@ jobs:
|
||||
sbom: true
|
||||
# Immutable tag only; branch tags are moved atomically for the
|
||||
# whole image set by the `promote` job.
|
||||
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:build-${{ needs.prepare.outputs.build_key }}
|
||||
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:sha-${{ needs.prepare.outputs.sha }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache
|
||||
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache,mode=max
|
||||
@@ -249,7 +245,7 @@ jobs:
|
||||
for image in $ALL_IMAGES; do
|
||||
docker buildx imagetools create \
|
||||
-t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \
|
||||
"${{ secrets.DOCKER_REGISTRY }}/$image:build-${{ needs.prepare.outputs.build_key }}"
|
||||
"${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.sha }}"
|
||||
done
|
||||
|
||||
# The marker is written LAST: its presence certifies that all five
|
||||
@@ -261,11 +257,11 @@ jobs:
|
||||
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
|
||||
run: |
|
||||
echo "${{ github.run_id }}" | aws s3 cp - \
|
||||
"s3://${{ secrets.S3_BUCKET }}/markers/images-${{ needs.prepare.outputs.build_key }}"
|
||||
"s3://${{ secrets.S3_BUCKET }}/markers/images-sha-${{ needs.prepare.outputs.sha }}"
|
||||
{
|
||||
echo "### ✅ Image set promoted"
|
||||
echo ""
|
||||
echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`build-${{ needs.prepare.outputs.build_key }}\`."
|
||||
echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.sha }}\`."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# ── 4. Single failure notification for the whole workflow ─────────────
|
||||
|
||||
@@ -102,17 +102,14 @@ jobs:
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
name: "Integration Tests (${{ matrix.shard }})"
|
||||
runs-on: penpot-extended-runner
|
||||
timeout-minutes: ${{ github.base_ref == 'staging' && 60 || 25 }}
|
||||
timeout-minutes: 40
|
||||
|
||||
needs: build-integration
|
||||
|
||||
# TEMPORARY (release stabilization): PRs targeting `staging` run on a
|
||||
# single serial shard, so new flakes cannot block the release work.
|
||||
# Remove the `github.base_ref` branch below to restore full parallelism.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: ${{ fromJSON(inputs.shards || (github.base_ref == 'staging' && '[1]' || '[1, 2, 3, 4]')) }}
|
||||
shard: ${{ fromJSON(inputs.shards || '[1, 2, 3, 4]') }}
|
||||
|
||||
container:
|
||||
image: penpotapp/devenv:latest
|
||||
@@ -153,12 +150,8 @@ jobs:
|
||||
working-directory: ./frontend
|
||||
env:
|
||||
WORKERS: ${{ inputs.workers }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
run: |
|
||||
# TEMPORARY (release stabilization): see the note on the matrix above.
|
||||
if [ -z "$WORKERS" ]; then
|
||||
if [ "$BASE_REF" = "staging" ]; then WORKERS=1; else WORKERS=2; fi
|
||||
fi
|
||||
WORKERS=${WORKERS:-2}
|
||||
echo "Running shard ${{ matrix.shard }}/${{ strategy.job-total }} with $WORKERS workers"
|
||||
pnpm exec playwright test --project default \
|
||||
--workers="$WORKERS" \
|
||||
@@ -236,6 +229,18 @@ jobs:
|
||||
fi
|
||||
jq -r -f ../.github/scripts/playwright-summary.jq report.json >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Kept for 30 days so flakiness rates can be aggregated across runs
|
||||
# without scraping job logs.
|
||||
- name: Upload JSON report
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: integration-json-report
|
||||
path: frontend/report.json
|
||||
overwrite: true
|
||||
if-no-files-found: ignore
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload HTML report
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
|
||||
@@ -24,6 +24,7 @@ opencode.json
|
||||
!AGENTS.md
|
||||
!CODE_OF_CONDUCT.md
|
||||
!SECURITY.md
|
||||
!HIGHLIGHTS.md
|
||||
/*.png
|
||||
/*.svg
|
||||
/*.sql
|
||||
@@ -98,6 +99,7 @@ opencode.json
|
||||
/.idea
|
||||
*.iml
|
||||
/.claude
|
||||
/CLAUDE.md
|
||||
/.playwright-mcp
|
||||
/.devenv/mcp/
|
||||
/opencode.json
|
||||
|
||||
@@ -3,11 +3,8 @@ description: Execute a ready plan end-to-end — create a GitHub issue, branch i
|
||||
agent: build
|
||||
---
|
||||
|
||||
# Implement Plan
|
||||
|
||||
This command is run once a plan is ready (for example, from plan mode). Execute
|
||||
the plan already prepared in the current session context — it does not take
|
||||
extra arguments. Follow these steps in order.
|
||||
the plan already prepared in the current session context. Follow these steps in order.
|
||||
|
||||
## 1. Create the issue
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
Act as a senior software engineer and perform a thorough review.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. **Determine what is being reviewed** from the provided context:
|
||||
- **If it is a plan** (implementation plan, design document, task breakdown) → load the **`plan-review`** skill.
|
||||
- **If it is code** (diff, PR, code change) → load the **`code-review`** skill.
|
||||
|
||||
2. Read `AGENTS.md` and follow its instructions for finding and reading all related testing documentation from memories before reviewing.
|
||||
|
||||
3. **Skip generated files, lockfile-only changes, and unrelated modifications** unless they introduce security risks.
|
||||
|
||||
4. Follow the loaded skill's process and produce its output format.
|
||||
|
||||
## Strong Rules
|
||||
|
||||
1. Do not invent problems. Every finding must be real and actionable.
|
||||
2. Do not modify any code and do not create a commit — this command only reviews.
|
||||
3. Be specific and constructive. "This could be better" is not helpful — explain why and how.
|
||||
4. Prioritize by impact. One structural issue outweighs ten nits.
|
||||
5. Missing tests are an issue, not a suggestion. Report as a severity-tagged finding — never as a recommendation.
|
||||
|
||||
## Context
|
||||
|
||||
$ARGUMENTS
|
||||
@@ -106,6 +106,8 @@ For detailed security guidance, see `security-and-hardening`.
|
||||
| **Low:** | Minor, optional | Author may ignore — formatting, style preferences |
|
||||
| **Suggestion:** | Worth considering | Not required, but improves the code |
|
||||
|
||||
**Unique finding IDs.** Assign every finding a stable identifier: `F1`, `F2`, `F3`, … numbered in order of severity (Critical first, then High, Medium, Low, Suggestion). Use the ID everywhere the finding is mentioned — in section headers, in the verdict, in follow-up discussion. Never renumber within a review. Example: `**F3 (High)** — `app/validate.cljs:42` — duplicate branch logic…`.
|
||||
|
||||
For each finding, describe the circumstances under which it could fail: specific inputs, load conditions, timing, or user actions that trigger the problem. "This crashes when input is null" is actionable; "this might crash" is not.
|
||||
|
||||
Lead with what matters: correctness and security first, then structural issues, then everything else. A few high-conviction comments beat a long list.
|
||||
@@ -122,11 +124,11 @@ Briefly explain what the code does and give an overall assessment.
|
||||
|
||||
### Critical and High-Priority Issues
|
||||
|
||||
List problems that could cause security incidents, data loss, crashes, incorrect behavior, or major performance degradation. For each: state the severity, identify the file/function/code section, explain why it's a problem, describe failure circumstances, and provide a concrete improvement with corrected code when useful.
|
||||
List problems that could cause security incidents, data loss, crashes, incorrect behavior, or major performance degradation. Each finding gets its unique ID (`F1`, `F2`, …). For each: state the severity, identify the file/function/code section, explain why it's a problem, describe failure circumstances, and provide a concrete improvement with corrected code when useful.
|
||||
|
||||
### Other Findings
|
||||
|
||||
List medium- and low-priority issues, including maintainability and design concerns.
|
||||
List medium- and low-priority issues, including maintainability and design concerns. Continue the ID sequence started above (`F3`, `F4`, …).
|
||||
|
||||
### Suggested Refactoring
|
||||
|
||||
@@ -148,6 +150,8 @@ Choose one:
|
||||
- **Approve with minor changes** — Good to merge after addressing low/medium issues
|
||||
- **Request changes** — Critical or high issues must be resolved before merge
|
||||
|
||||
List the finding IDs the verdict depends on (e.g. "Request changes: F1, F4").
|
||||
|
||||
## Change Sizing
|
||||
|
||||
Small, focused changes are easier to review, faster to merge, and safer to deploy.
|
||||
@@ -231,25 +235,13 @@ For supply-chain risk triage, follow the `security-and-hardening` skill.
|
||||
|
||||
## Verification
|
||||
|
||||
After review is complete:
|
||||
Before emitting the verdict, verify the change as it stands. This is the reviewer's own due diligence — it covers the state of the code at review time, not the later resolution of findings (fixing findings is the author's job; confirming them is a new review):
|
||||
|
||||
- [ ] All Critical issues are resolved
|
||||
- [ ] All Required (no-prefix) changes are resolved or explicitly deferred with justification
|
||||
- [ ] Tests pass
|
||||
- [ ] Tests pass — run them yourself, don't trust the claim
|
||||
- [ ] Build succeeds
|
||||
- [ ] The verification story is documented (what changed, how it was verified)
|
||||
- [ ] Dependency upgrades reviewed against changelog, isolated per package, verified by green suite
|
||||
|
||||
## Multi-Model Review Pattern
|
||||
|
||||
Use different models for different review perspectives:
|
||||
|
||||
```
|
||||
Model A writes the code → Model B reviews → Model A addresses feedback → Human makes the final call
|
||||
```
|
||||
|
||||
Different models have different blind spots.
|
||||
|
||||
## See Also
|
||||
|
||||
- For detailed security review guidance, see `security-and-hardening`
|
||||
@@ -1,13 +1,13 @@
|
||||
---
|
||||
name: planner
|
||||
description: Read-only planning and architecture analysis for Penpot — produce a structured implementation plan (Context, Affected modules, Approach, Risks, Testing). Always output to the user; additionally save to .opencode/plans/YYYY-MM-DD-<title>.md.
|
||||
description: Read-only planning and architecture analysis for Penpot — produce a structured implementation plan with task breakdown, acceptance criteria, sizing, and checkpoints. Always output to the user and save to .opencode/plans/YYYY-MM-DD-<title>.md.
|
||||
---
|
||||
|
||||
# Planner
|
||||
|
||||
Read-only senior software architect role for Penpot. Produces structured
|
||||
implementation plans that engineers or other agents can execute. Never writes
|
||||
or modifies code.
|
||||
implementation plans with task breakdowns that engineers or other agents can
|
||||
execute. Never writes or modifies code.
|
||||
|
||||
## When to Use
|
||||
|
||||
@@ -18,24 +18,29 @@ or modifies code.
|
||||
- The user asks "how would I implement X?" or "what's involved in fixing Y?".
|
||||
- The user is about to start non-trivial work and wants a bite-sized task
|
||||
breakdown.
|
||||
- A task feels too large or vague to start.
|
||||
- Work needs to be parallelized across multiple agents or sessions.
|
||||
|
||||
Do **not** use this skill to actually implement anything — it is read-only.
|
||||
|
||||
**When NOT to use:** Single-file changes with obvious scope, or when the spec
|
||||
already contains well-defined tasks.
|
||||
|
||||
## Role
|
||||
|
||||
You are a Senior Software Architect working on Penpot, an open-source design
|
||||
tool. Your sole responsibility is planning and analysis — you do NOT write or
|
||||
modify code.
|
||||
You help users understand the Penpot codebase, design solutions, and produce
|
||||
implementation plans that other agents or developers can execute. The plan
|
||||
tells them what to build and how to verify it, task by task.
|
||||
|
||||
You help users understand the codebase, design solutions, and create detailed
|
||||
implementation plans that other agents or developers can execute. Document
|
||||
everything they need to know: which files to touch for each task, code patterns,
|
||||
tests, and how to verify correctness. Apply DRY and KISS principles.
|
||||
The implementer reads the project's agent docs (`AGENTS.md`, project memories
|
||||
such as `mem:critical-info`, `mem:testing`, and each module's core memory)
|
||||
before working. Reference those memories instead of re-explaining tooling,
|
||||
conventions, or test design — explain in the plan only what they do not cover.
|
||||
|
||||
Do **not** suggest commit messages or commit names anywhere in your plans or
|
||||
responses — committing is the developer's responsibility.
|
||||
responses — committing is the implementer's responsibility.
|
||||
|
||||
## Required Reading Before Planning
|
||||
## CRITICAL: Required Reading Before Planning
|
||||
|
||||
Before drafting any plan, work through the project's own guidance:
|
||||
|
||||
@@ -50,6 +55,8 @@ Before drafting any plan, work through the project's own guidance:
|
||||
|
||||
Skipping this step is the #1 cause of incorrect or incomplete plans.
|
||||
|
||||
---
|
||||
|
||||
## The Planning Process
|
||||
|
||||
### Phase 1: Architecture Analysis
|
||||
@@ -64,16 +71,42 @@ Skipping this step is the #1 cause of incorrect or incomplete plans.
|
||||
|
||||
### Phase 2: Task Breakdown
|
||||
|
||||
Implementation order follows the monorepo's dependency graph:
|
||||
`frontend -> common`, `backend -> common`, `exporter -> common`,
|
||||
`frontend -> render-wasm`. Build shared foundations first, then layer
|
||||
consumers on top.
|
||||
#### Identify the Dependency Graph
|
||||
|
||||
Map what depends on what, following the monorepo's module dependency graph:
|
||||
|
||||
```
|
||||
common (shared types, schemas — no deps)
|
||||
│
|
||||
├── backend (depends common)
|
||||
│ ├── RPC handlers
|
||||
│ └── persistence / migrations
|
||||
│
|
||||
├── frontend (depends common, render-wasm)
|
||||
│ ├── UI components
|
||||
│ └── state / API integration
|
||||
│
|
||||
├── exporter (depends common)
|
||||
│
|
||||
└── render-wasm (consumed by frontend)
|
||||
```
|
||||
|
||||
Implementation order follows the dependency graph bottom-up: build shared
|
||||
foundations first, then layer consumers on top.
|
||||
|
||||
#### Slice Vertically
|
||||
|
||||
Instead of building all of common, then all of backend, then all of frontend —
|
||||
build one complete feature path at a time:
|
||||
|
||||
**Bad (horizontal slicing):**
|
||||
```
|
||||
Task 1: Build all common types
|
||||
Task 2: Build all backend handlers
|
||||
Task 3: Build all frontend components
|
||||
```
|
||||
|
||||
**Good (vertical slicing):**
|
||||
```
|
||||
Task 1: common data types + schema ← foundation
|
||||
Task 2: backend RPC handler + persistence
|
||||
@@ -89,39 +122,58 @@ Each task follows this structure:
|
||||
```markdown
|
||||
## Task [N]: [Short descriptive title]
|
||||
|
||||
**Description:** One paragraph explaining what this task accomplishes.
|
||||
**Description:** One or two paragraphs explaining what this task accomplishes.
|
||||
Should be clear and concise.
|
||||
|
||||
**Rationale:** Why this task exists and why this approach over the obvious
|
||||
alternatives — design decisions, trade-offs, constraints discovered during
|
||||
analysis. One or two sentences; skip only if genuinely trivial.
|
||||
|
||||
**Code sketch (optional):** Signature-, type-, or shape-level example when the
|
||||
intended interface is non-obvious. Keep it short — a skeleton that fixes the
|
||||
contract (function signature, model fields, error shape), never a full
|
||||
implementation. Omit when the task is mechanical.
|
||||
|
||||
**Acceptance criteria:**
|
||||
- [ ] [Specific, testable condition]
|
||||
- [ ] [Specific, testable condition]
|
||||
|
||||
**Verification:**
|
||||
- [ ] Tests pass (module-specific test command)
|
||||
- [ ] Lint/formatter passes (module-specific check command)
|
||||
- [ ] Relevant tests pass (module-specific test command).
|
||||
- [ ] Lint/formatter passes (module-specific check command), if applicable.
|
||||
- [ ] The core flow works end-to-end, if applicable.
|
||||
|
||||
**Dependencies:** [Task numbers this depends on, or "None"]
|
||||
|
||||
**Files likely touched:**
|
||||
- `path/to/file.clj`
|
||||
- `path/to/file_test.clj`
|
||||
|
||||
**Estimated scope:** [XS: 1 file | S: 1-2 files | M: 3-5 files | L: 5+ files]
|
||||
```
|
||||
|
||||
Replace "module-specific test command" with the actual commands for the module
|
||||
(e.g. `clojure -M:dev:test` for backend/common, `npx shadow-cljs compile test && npx karma start` for frontend,
|
||||
or the commands noted in the module's core memory).
|
||||
(e.g. `clojure -M:dev:test` for backend/common,
|
||||
`npx shadow-cljs compile test && npx karma start` for frontend, or the
|
||||
commands noted in the module's core memory).
|
||||
|
||||
When possible, design each task with TDD in mind: acceptance criteria double
|
||||
as a test list, and the natural first step of the task is writing those tests
|
||||
before the implementation. Some tasks resist this (config, migrations, pure
|
||||
wiring) — for those, keep the usual verification steps.
|
||||
|
||||
#### Estimate Scope
|
||||
|
||||
| Size | Files | Scope |
|
||||
|------|-------|-------|
|
||||
| **XS** | 1 | Single function, config change, or schema tweak |
|
||||
| **S** | 1-2 | One handler or component method |
|
||||
| **M** | 3-5 | One vertical feature slice |
|
||||
| **L** | 5-8 | Multi-component feature |
|
||||
| **XL** | 8+ | **Too large — break it down further** |
|
||||
| Size | Files | Scope | Example |
|
||||
|------|-------|-------|---------|
|
||||
| **XS** | 1 | Single function, config change, or schema tweak | Add a validation rule |
|
||||
| **S** | 1-2 | One handler or component method | Add a new RPC endpoint |
|
||||
| **M** | 3-5 | One vertical feature slice | Bookmark CRUD with tests |
|
||||
| **L** | 5-8 | Multi-component feature | Search with filtering and pagination |
|
||||
| **XL** | 8+ | **Too large — break it down further** | — |
|
||||
|
||||
If a task is L or larger, break it into smaller tasks. Agents perform best on
|
||||
S and M tasks.
|
||||
If a task is XL, it should be broken into smaller tasks. Agents perform best
|
||||
on S and M tasks.
|
||||
|
||||
**When to break a task down further:**
|
||||
- It would take more than one focused session
|
||||
@@ -141,11 +193,11 @@ Arrange tasks so that:
|
||||
Add explicit checkpoints with the relevant module commands:
|
||||
|
||||
```markdown
|
||||
## Checkpoint: After Tasks 1-3
|
||||
- [ ] All tests pass (module-specific command)
|
||||
- [ ] Lint/format passes (module-specific command)
|
||||
- [ ] Core flow works end-to-end
|
||||
- [ ] Review with human before proceeding
|
||||
### Checkpoint: After Tasks 1-3
|
||||
- [ ] Relevant tests pass (module-specific command).
|
||||
- [ ] The relevant build or compilation passes, if applicable.
|
||||
- [ ] The core flow works end-to-end.
|
||||
- [ ] Review with human before proceeding.
|
||||
```
|
||||
|
||||
## Requirements
|
||||
@@ -159,7 +211,7 @@ Add explicit checkpoints with the relevant module commands:
|
||||
- Apply DRY and KISS principles to the proposed implementation.
|
||||
- Define a testing strategy aligned with each affected module's tooling.
|
||||
- Every task must have acceptance criteria and verification steps.
|
||||
- Checkpoints must exist between major phases.
|
||||
- Checkpoints must exist after every 2-3 tasks.
|
||||
|
||||
## Constraints
|
||||
|
||||
@@ -168,7 +220,8 @@ Add explicit checkpoints with the relevant module commands:
|
||||
`.opencode/plans/`.
|
||||
- You do **not** run builds, tests, linters, or any commands that modify state.
|
||||
- You do **not** create git commits or interact with version control.
|
||||
- You do **not** execute shell commands beyond read-only searches.
|
||||
- You do **not** execute shell commands beyond read-only searches (`rg`, `ls`,
|
||||
`find`, `cat`, `bat`).
|
||||
- Your output is a structured plan or analysis, ready for handoff to an
|
||||
engineer agent or developer.
|
||||
|
||||
@@ -188,8 +241,9 @@ slug is lowercase, hyphen-separated, and a short summary of the task
|
||||
(e.g. `add-batch-get-profiles-for-file-comments`). Create the
|
||||
`.opencode/plans/` directory if it does not exist.
|
||||
|
||||
Always attempt the write. If the user explicitly provides a target file path,
|
||||
use that path instead of the default.
|
||||
IMPORTANT: The plan agent has write permission specifically for
|
||||
`.opencode/plans/` — always attempt the write. If the user explicitly provides
|
||||
a target file path, use that path instead of the default.
|
||||
|
||||
### Plan Document Template
|
||||
|
||||
@@ -212,41 +266,75 @@ use that path instead of the default.
|
||||
security implications.]
|
||||
|
||||
## Approach
|
||||
[Step-by-step implementation plan with file paths, function names, and code
|
||||
shape where applicable. Group steps into atomic, ordered tasks.]
|
||||
[A short strategy summary: 3-5 sentences describing the overall approach and
|
||||
the shape of the dependency graph (what depends on what, what gets built
|
||||
first). High-level only — the task-by-task detail lives in the Task List.]
|
||||
|
||||
## Task List
|
||||
|
||||
### Phase 1: Foundation
|
||||
- [ ] Task 1: ...
|
||||
- [ ] Task 2: ...
|
||||
Each task uses the full task structure defined in
|
||||
[Write Tasks](#write-tasks) — description, rationale, acceptance criteria,
|
||||
verification, dependencies, files, estimated scope, and optional code sketch.
|
||||
Never reduce a task to a one-line checkbox; the plan must be self-contained
|
||||
and executable without other context.
|
||||
|
||||
### Checkpoint: Phase 1
|
||||
- [ ] Tests pass, lint/formatter clean (module-specific commands)
|
||||
Tasks are a flat, ordered list — a plan is not a roadmap. Do not group tasks
|
||||
into phases, milestones, or sprints; ordering and dependencies are already
|
||||
captured per task. Insert a checkpoint after every 2-3 tasks.
|
||||
|
||||
### Phase 2: Core Features
|
||||
- [ ] Task 3: ...
|
||||
- [ ] Task 4: ...
|
||||
## Task 1: [Short descriptive title]
|
||||
|
||||
### Checkpoint: Phase 2
|
||||
- [ ] End-to-end flow works
|
||||
**Description:** [What this task accomplishes.]
|
||||
|
||||
### Phase 3: Polish
|
||||
- [ ] Task 5: ...
|
||||
- [ ] Task 6: ...
|
||||
**Rationale:** [Why this approach over the alternatives.]
|
||||
|
||||
### Checkpoint: Complete
|
||||
- [ ] All acceptance criteria met
|
||||
- [ ] Ready for review
|
||||
**Acceptance criteria:**
|
||||
- [ ] [Specific, testable condition]
|
||||
|
||||
## Testing Strategy
|
||||
[How to verify: which test commands to run per module, what cases to cover,
|
||||
manual verification steps, lint/format checks. Consult each module's core
|
||||
memory for the exact commands.]
|
||||
**Verification:**
|
||||
- [ ] Relevant tests pass (module-specific command).
|
||||
|
||||
**Dependencies:** None
|
||||
|
||||
**Files likely touched:**
|
||||
- `path/to/file`
|
||||
|
||||
**Estimated scope:** [XS: 1 file | S: 1-2 files | M: 3-5 files | L: 5+ files]
|
||||
|
||||
**Code sketch (optional):** [Short contract-level example, only if the shape
|
||||
is non-obvious.]
|
||||
|
||||
## Task 2: [Short descriptive title]
|
||||
|
||||
[Same structure as Task 1.]
|
||||
|
||||
## Task 3: [Short descriptive title]
|
||||
|
||||
[Same structure as Task 1.]
|
||||
|
||||
### Checkpoint: After Tasks 1-3
|
||||
- [ ] Relevant tests pass (module-specific command).
|
||||
- [ ] The relevant build or compilation passes, if applicable.
|
||||
- [ ] The core flow works end-to-end.
|
||||
- [ ] Review with human before proceeding.
|
||||
|
||||
## Task 4: [Short descriptive title]
|
||||
|
||||
[Same structure as Task 1.]
|
||||
|
||||
## Task 5: [Short descriptive title]
|
||||
|
||||
[Same structure as Task 1.]
|
||||
|
||||
## Verification & Testing
|
||||
[How to verify each task and the whole plan: the project's real test, lint,
|
||||
build, and run commands (extracted during Required Reading), coverage
|
||||
expectations, and manual checks. Consult each module's core memory for the
|
||||
exact commands.]
|
||||
|
||||
## Parallelization Opportunities
|
||||
- **Safe to parallelize:** Independent feature slices across separate
|
||||
modules, tests for already-implemented features
|
||||
modules, tests for already-implemented features, documentation
|
||||
- **Must be sequential:** Shared common schema changes, database migrations
|
||||
- **Needs coordination:** Features that share a contract (define the contract
|
||||
first, then parallelize)
|
||||
@@ -259,13 +347,31 @@ When the plan is purely analytical (e.g. a code review or feasibility study
|
||||
with no implementation), skip the **Approach** and **Task List** sections and
|
||||
lead with **Findings** instead, keeping the rest of the structure.
|
||||
|
||||
## Common Rationalizations
|
||||
|
||||
| Rationalization | Reality |
|
||||
|---|---|
|
||||
| "I'll figure it out as I go" | That's how you end up with a tangled mess and rework. 10 minutes of planning saves hours. |
|
||||
| "The tasks are obvious" | Write them down anyway. Explicit tasks surface hidden dependencies and forgotten edge cases. |
|
||||
| "Planning is overhead" | Planning is the task. Implementation without a plan is just typing. |
|
||||
| "I can hold it all in my head" | Context windows are finite. Written plans survive session boundaries and compaction. |
|
||||
|
||||
## Red Flags
|
||||
|
||||
- Delivering prose without a task breakdown
|
||||
- Tasks that say "implement the feature" without acceptance criteria
|
||||
- No verification steps in the plan
|
||||
- All tasks are XL-sized
|
||||
- No checkpoints between tasks
|
||||
- Dependency order isn't considered
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
Before starting implementation, confirm:
|
||||
Before delivering the plan, confirm:
|
||||
|
||||
- [ ] Every task has acceptance criteria
|
||||
- [ ] Every task has a verification step
|
||||
- [ ] Task dependencies are identified and ordered correctly
|
||||
- [ ] No task touches more than ~5 files
|
||||
- [ ] Checkpoints exist between major phases
|
||||
- [ ] The human has reviewed and approved the plan
|
||||
- [ ] No task is XL or larger — break it down instead
|
||||
- [ ] Checkpoints exist after every 2-3 tasks
|
||||
- [ ] The plan is ready for human review
|
||||
@@ -477,9 +477,15 @@ There are exactly two types:
|
||||
release, but the PR is being released elsewhere — the fix may not
|
||||
actually ship here.
|
||||
2. **PR is in the milestone, but the issue it closes is in a different
|
||||
milestone (or has no milestone).** The PR is being released here, but
|
||||
the issue it fixes is being released in a different version (or never
|
||||
tracked in a milestone) — the changelog pairing is misleading.
|
||||
milestone.** The PR is being released here, but the issue it fixes is
|
||||
being released in a different version — the changelog pairing is
|
||||
misleading.
|
||||
|
||||
**Exception — issue with no milestone is NOT an anomaly.** Milestones
|
||||
are only required for issues tracked in the "Main" project. A milestone
|
||||
PR that closes an issue with no milestone references an issue from
|
||||
another (probably private) project; that is expected and the issue is
|
||||
not part of this changelog. Do not report it.
|
||||
|
||||
**Anything else is not an anomaly.** Other discrepancies (exclusion
|
||||
labels on in-changelog issues, missing valid issues, unmerged PR
|
||||
@@ -635,6 +641,10 @@ for pr_num in sorted(changelog_prs):
|
||||
if get_pr_milestone(pr_num) != MILESTONE: continue
|
||||
for issue_num in pr.get('closing_issues', []):
|
||||
issue_ms = get_issue_milestone(issue_num)
|
||||
# No milestone = issue from another (probably private) project —
|
||||
# milestones are only required for the "Main" project. Not an
|
||||
# anomaly, and the issue never belongs in this changelog.
|
||||
if issue_ms is None: continue
|
||||
if issue_ms != MILESTONE:
|
||||
anomalies_b.append({
|
||||
'pr': pr_num,
|
||||
@@ -657,7 +667,7 @@ with open(OUTPUT, 'w') as f:
|
||||
|
||||
f.write('## Summary\n\n')
|
||||
f.write(f'- **Issue in {MILESTONE}, referenced PR in different milestone or no milestone:** {n_a}\n')
|
||||
f.write(f'- **PR in {MILESTONE}, closing issue in different milestone or no milestone:** {n_b}\n')
|
||||
f.write(f'- **PR in {MILESTONE}, closing issue in a different milestone:** {n_b}\n')
|
||||
f.write(f'- **Total anomalies:** {n_a + n_b}\n\n')
|
||||
|
||||
# --- Anomalies section ---
|
||||
@@ -686,7 +696,7 @@ with open(OUTPUT, 'w') as f:
|
||||
f.write('\n')
|
||||
|
||||
if n_b:
|
||||
f.write(f'\n### PR in {MILESTONE}, closing issue in different milestone or no milestone\n\n')
|
||||
f.write(f'\n### PR in {MILESTONE}, closing issue in a different milestone\n\n')
|
||||
by_pr = {}
|
||||
for b in anomalies_b:
|
||||
by_pr.setdefault(b['pr'], []).append(b)
|
||||
@@ -721,8 +731,11 @@ milestone mismatches between issues and their referenced PRs:
|
||||
|
||||
1. **Issue in milestone, referenced PR in different milestone or no milestone** —
|
||||
the changelog claims a fix here, but the PR is released elsewhere.
|
||||
2. **PR in milestone, closing issue in different milestone or no milestone** —
|
||||
2. **PR in milestone, closing issue in a different milestone** —
|
||||
the PR is released here, but the issue it fixes belongs to another version.
|
||||
(An issue with *no* milestone belongs to another, probably private,
|
||||
project — milestones are only required on the "Main" project — so it is
|
||||
neither an anomaly nor a changelog candidate.)
|
||||
|
||||
**Rule violations are not in the report** — they are workflow errors the
|
||||
LLM must fix directly in `CHANGES.md` during step 6a (pre-flight checks).
|
||||
@@ -799,8 +812,11 @@ self-contained and clickable in any Markdown viewer.
|
||||
- **Anomaly = milestone mismatch only.** The report contains only milestone
|
||||
mismatches: (1) the issue is in this milestone but the referenced PR is
|
||||
in a different milestone (or unassigned), and (2) the PR is in this
|
||||
milestone but the issue it closes is in a different milestone (or
|
||||
unassigned). These are anomalies because the changelog pairing is
|
||||
milestone but the issue it closes is in a different milestone. An
|
||||
*unassigned* (milestone-less) issue closed by a milestone PR is **not**
|
||||
an anomaly: milestones are required only for the "Main" project, so such
|
||||
issues come from another (probably private) project and are not changelog
|
||||
candidates. These anomalies are reported because the changelog pairing is
|
||||
*misleading* — the human needs to decide whether the milestone or the
|
||||
changelog is wrong. All other discrepancies (exclusion labels, missing
|
||||
valid issues, unmerged PR references, duplicates, stale milestone
|
||||
|
||||
@@ -7,6 +7,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m
|
||||
- RPC, DB helpers, workers, cron: `mem:backend/rpc-db-worker-subtleties`
|
||||
- Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`.
|
||||
- HTTP sessions, config, media processing, and file data persistence: `mem:backend/http-storage-filedata-subtleties`.
|
||||
- Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment`
|
||||
- Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains`
|
||||
- Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`.
|
||||
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
# Graph Experiment
|
||||
|
||||
## Scope
|
||||
|
||||
- Purpose: project Penpot file data into an embedded Ladybug graph database.
|
||||
- Purpose: keep the graph current with Penpot file changes.
|
||||
- Purpose: expose a read-only graph console for backend debugging.
|
||||
- This is an experiment, not a replacement for PostgreSQL file storage.
|
||||
- The graph subsystem is off unless `:graph` is in the backend flags.
|
||||
- The main Penpot frontend has no graph feature code for this subsystem.
|
||||
- The graph console is a backend-served HTML template with JavaScript.
|
||||
|
||||
## Memory Links
|
||||
|
||||
- Read `mem:backend/core` for backend architecture, HTTP routes, DB rules, and test commands.
|
||||
- Read `mem:backend/rpc-db-worker-subtleties` for RPC and message bus behavior.
|
||||
- Read `mem:backend/http-storage-filedata-subtleties` for file data loading and realization.
|
||||
- Read `mem:common/changes-architecture` for the change record vocabulary.
|
||||
- Read `mem:frontend/routing-app-shell-subtleties` for the existing notification WebSocket.
|
||||
- Read `mem:prod-infra/core` for Redis or Valkey message bus topology.
|
||||
|
||||
## Branch Surface
|
||||
|
||||
- The graph experiment adds about 6,336 lines and changes about 27 files.
|
||||
- The graph implementation lives under `backend/src/app/graph/`.
|
||||
- The graph console lives at `backend/resources/app/templates/graph-console.tmpl`.
|
||||
- The existing debug page gains graph links in `backend/resources/app/templates/debug.tmpl`.
|
||||
- The existing debug HTTP routes gain graph handlers in `backend/src/app/http/debug.clj`.
|
||||
- The backend system passes the message bus to the debug route component in `backend/src/app/main.clj`.
|
||||
- The backend adds Ladybug and Arrow dependencies in `backend/deps.edn`.
|
||||
- The backend adds JVM options for Ladybug and Arrow native access.
|
||||
- The common flag registry adds `:graph` in `common/src/app/common/flags.cljc`.
|
||||
- The graph experiment adds `graph_sync_parity_test.clj` and `graph_binder_gate_test.clj`.
|
||||
|
||||
## System Model
|
||||
|
||||
### Storage layers
|
||||
|
||||
- PostgreSQL remains the source of truth for Penpot files.
|
||||
- The graph database stores a projection of one file.
|
||||
- A persistent graph uses a `.lbug` path under `PENPOT_GRAPH_DIR`.
|
||||
- The default graph directory is `/tmp/penpot-graph`.
|
||||
- A debug session uses a Ladybug `:memory:` database.
|
||||
- A debug session database lives inside the backend JVM process.
|
||||
- A debug session does not survive a backend restart.
|
||||
- A debug session does not store file data back to PostgreSQL.
|
||||
|
||||
### Two graph update paths
|
||||
|
||||
- Cold projection reads the complete file and rebuilds the graph.
|
||||
- Incremental sync reads file change records and updates the open graph.
|
||||
- Both paths must produce the same graph for the same file state.
|
||||
- The parity test treats cold projection as the reference path.
|
||||
- A reload discards the session graph and uses cold projection again.
|
||||
|
||||
## Main Namespaces
|
||||
|
||||
### `app.graph.ladybug`
|
||||
|
||||
- Opens and closes Ladybug `Database` and `Connection` objects.
|
||||
- Installs and loads the Ladybug JSON extension.
|
||||
- Executes Cypher statements.
|
||||
- Executes prepared statements.
|
||||
- Binds scalar parameters.
|
||||
- Formats UUID, string, integer, number, JSON, and timestamp values.
|
||||
- Formats compound values such as arrays, maps, and structs.
|
||||
- Converts Ladybug values back to Clojure values.
|
||||
- Limits normal query results to 200 rows by default.
|
||||
- Detects result truncation with `:truncated?`.
|
||||
- Uses query timeout `0` by default.
|
||||
- Query timeout `0` disables the timeout.
|
||||
- Provides `validate-on-connection!` for parse, bind, and read-only checks.
|
||||
- `exec-prepared-on-connection!` prepares every statement before the first execution.
|
||||
- A prepare failure stops the batch before a mutation runs.
|
||||
|
||||
### `app.graph.schema`
|
||||
|
||||
- Provides the public schema facade.
|
||||
- Exposes schema version `penpot-graph-slice-4`.
|
||||
- Delegates node and relationship definitions to `app.graph.schema.nodes`.
|
||||
|
||||
### `app.graph.schema.nodes`
|
||||
|
||||
- Holds the single registry for graph node tables.
|
||||
- Generates node DDL.
|
||||
- Generates relationship DDL.
|
||||
- Maps Penpot shape types to graph tables.
|
||||
- Projects source attributes into graph attributes.
|
||||
- Formats graph column values.
|
||||
- Quotes reserved graph labels such as `Group` and `Boolean`.
|
||||
- Defines container tables and shape tables.
|
||||
- Defines `IsChildOf`, `IsInstanceOf`, `RefersTo`, and `FillsSwapSlot`.
|
||||
|
||||
### `app.graph.schema.contract`
|
||||
|
||||
- Records deliberate graph contract decisions.
|
||||
- Renames graph columns such as `:revn` to `revision`.
|
||||
- Drops attributes that do not belong in this graph slice.
|
||||
- Records attributes that the graph does not project.
|
||||
- Applies per-table dropped attributes.
|
||||
- Defines type overrides for vectors, transforms, colors, maps, and JSON arrays.
|
||||
- Maps selected map keys to the frontend JSON naming convention.
|
||||
- `:background-blur` remains a declared unprojected attribute.
|
||||
|
||||
### `app.graph.schema.projection`
|
||||
|
||||
- Derives projected schemas from canonical Malli schemas.
|
||||
- Builds the projected document schema.
|
||||
- Builds projected shape schemas.
|
||||
- Selects the schema for each shape type.
|
||||
|
||||
### `app.graph.schema.types`
|
||||
|
||||
- Maps Malli types to Ladybug types.
|
||||
- Maps matrices to `DOUBLE[6]`.
|
||||
- Maps points to `DOUBLE[2]`.
|
||||
- Maps rectangles to `DOUBLE[4]`.
|
||||
- Maps colors to `UINT32`.
|
||||
- Maps collections to Ladybug arrays.
|
||||
- Maps `:map-of` schemas to `MAP`.
|
||||
- Maps closed scalar maps to `STRUCT`.
|
||||
- Maps other complex values to `JSON`.
|
||||
|
||||
### `app.graph.schema.values`
|
||||
|
||||
- Coerces source values to graph column values.
|
||||
- Writes fixed vectors with deterministic order.
|
||||
- Packs colors into the graph color representation.
|
||||
- Sorts set values when deterministic output is needed.
|
||||
|
||||
### `app.graph.arrow`
|
||||
|
||||
- Loads projection rows with Apache Arrow.
|
||||
- Creates temporary staged node and relationship tables.
|
||||
- Uses `COPY ... FROM (MATCH ...)` for bulk loading.
|
||||
- Groups relationship loads by source and target table pair.
|
||||
- Resolves relationship endpoints with joins.
|
||||
- Does not use `createArrowRelTable` for UUID relationship endpoints.
|
||||
- Keeps the Arrow `RootAllocator` alive until Ladybug releases staged buffers.
|
||||
- Closes the allocator after the connection and database close sequence.
|
||||
|
||||
### `app.graph.ingest`
|
||||
|
||||
- Fetches a complete file with `bfc/get-file` and `:realize? true`.
|
||||
- Rejects missing files.
|
||||
- Rejects files without file data.
|
||||
- Can run file data validation before projection.
|
||||
- Creates the DDL.
|
||||
- Loads nodes and edges through Arrow.
|
||||
- Executes post-load transforms.
|
||||
- Writes graph metadata last.
|
||||
- Treats the final metadata write as the complete-build marker.
|
||||
- Supports a persistent database path and an open connection.
|
||||
|
||||
### `app.graph.projection.document`
|
||||
|
||||
- Projects `Document`, `Page`, `Component`, and supported shape nodes.
|
||||
- Skips the page root frame.
|
||||
- Creates `IsChildOf` edges from shapes to parents.
|
||||
- Creates page edges to the document.
|
||||
- Creates component edges to the document.
|
||||
- Stores page order in `Page.index` and edge `position`.
|
||||
- Reverses the stored `:shapes` list for Penpot z-order.
|
||||
- Adds `page-id` to every projected shape.
|
||||
- Propagates an instance head `component-id` to descendants.
|
||||
- Stops component inheritance at a non-Frame shape with its own component ID.
|
||||
- Skips deleted components during cold projection.
|
||||
- Logs unsupported shape types and missing shape records.
|
||||
|
||||
### `app.graph.projection.transforms`
|
||||
|
||||
- Runs after the base nodes and edges load.
|
||||
- `link-component-instances` creates `IsInstanceOf` edges.
|
||||
- A Frame needs `component-file` to qualify as an instance head.
|
||||
- `link-shape-refs` creates `RefersTo` edges from `shape-ref`.
|
||||
- Ladybug limits multi-label relationship `MERGE` statements.
|
||||
- The transform emits one statement for each shape-table pair.
|
||||
- `link-swap-slots` creates `FillsSwapSlot` edges.
|
||||
- Swap slot IDs come from `swap-slot-<uuid>` entries in `touched`.
|
||||
- The transform removes swap slot entries from `touched` after edge creation.
|
||||
- The transform order matters because it reads and then changes `touched`.
|
||||
|
||||
### `app.graph.meta`
|
||||
|
||||
- Stores graph provenance in `GraphMeta`.
|
||||
- Stores schema version, source revision, producer, and build time.
|
||||
- The source revision identifies the file revision used for cold projection.
|
||||
|
||||
### `app.graph.stats` and `app.graph.report`
|
||||
|
||||
- `app.graph.stats` counts graph nodes and relationships from the live catalog.
|
||||
- `app.graph.report` prints ingest information for REPL use.
|
||||
|
||||
## Cold Projection Flow
|
||||
|
||||
1. Get the file row and realized file data from PostgreSQL.
|
||||
2. Read the file revision from the file row.
|
||||
3. Build the node and edge projection.
|
||||
4. Create all graph tables from the graph schema.
|
||||
5. Load node rows with Arrow.
|
||||
6. Load relationship rows with Arrow.
|
||||
7. Run `CHECKPOINT;`.
|
||||
8. Run the registered derived transforms.
|
||||
9. Write `GraphMeta` as the final build step.
|
||||
10. Return file ID, file revision, database path, projection stats, and transform stats.
|
||||
|
||||
### Projection node groups
|
||||
|
||||
- `Document` contains file-level attributes without the file data blob.
|
||||
- `Document.options` receives file-level options from the data blob.
|
||||
- `Page` contains page attributes without the page object map.
|
||||
- `Component` contains component attributes without component object maps.
|
||||
- Shape tables contain the supported shape attributes.
|
||||
- The graph stores selected derived attributes such as `page-id`.
|
||||
|
||||
### Projection relationship groups
|
||||
|
||||
- Structural edges use `IsChildOf`.
|
||||
- Page and component edges point to `Document`.
|
||||
- Derived edges come from the post-load transform registry.
|
||||
|
||||
## Incremental Sync
|
||||
|
||||
### Change source
|
||||
|
||||
- `app.rpc.commands.files-update` persists the file update first.
|
||||
- The same command publishes a `:file-change` message to the file topic.
|
||||
- The topic key is the file UUID.
|
||||
- The message contains the file ID, profile ID, session ID, revision, version, and changes.
|
||||
- Library changes also publish a team-topic message.
|
||||
- The graph session only consumes the file-topic `:file-change` messages.
|
||||
|
||||
### Session subscription
|
||||
|
||||
- `app.graph.debug/start-sync-loop!` creates a channel with a dropping buffer of 64.
|
||||
- The session subscribes the channel to the file UUID topic.
|
||||
- The loop reads one message at a time.
|
||||
- The loop ignores message types other than `:file-change`.
|
||||
- The loop stops when the channel closes.
|
||||
- `destroy-session!` closes the channel and purges its message bus subscription.
|
||||
|
||||
### Session state
|
||||
|
||||
- Sessions are stored in a global `defonce` atom.
|
||||
- The map key is the string form of `profile-id`.
|
||||
- One profile has one graph session.
|
||||
- Loading another file first destroys the old session.
|
||||
- A session stores the Ladybug database and connection.
|
||||
- A session stores a shared lock for graph access.
|
||||
- A session stores file metadata.
|
||||
- A session stores the incremental sync index.
|
||||
- A session stores the message bus channel.
|
||||
- A session stores load time and profile ID.
|
||||
- The session keeps projection statistics but drops full projection rows after index creation.
|
||||
|
||||
### Sync index
|
||||
|
||||
- `build-index` starts from the complete cold projection.
|
||||
- The index stores the graph file ID and document ID.
|
||||
- The index stores the current graph revision.
|
||||
- The index stores page IDs, names, and positions.
|
||||
- The index stores component IDs, names, and deleted state.
|
||||
- The index stores shape table, parent, position, frame, page, and component context.
|
||||
- The index stores child IDs by parent ID.
|
||||
- The index supports later change application without another PostgreSQL file read.
|
||||
|
||||
### Change application
|
||||
|
||||
- `apply-changes!` processes the change list in source order.
|
||||
- Each supported change returns a new index and a list of Cypher statements.
|
||||
- Unsupported changes enter the `:skipped` result.
|
||||
- Supported changes enter the `:applied` result.
|
||||
- The function collects all statements before it executes them.
|
||||
- The function appends a document revision statement when at least one change applies.
|
||||
- The index revision advances only when at least one change applies.
|
||||
- A larger incoming revision than the index revision creates a warning.
|
||||
- A revision gap does not trigger catch-up.
|
||||
|
||||
### Shape change rules
|
||||
|
||||
- `:add-obj` reuses `projection.document/denormalized-shape`.
|
||||
- `:add-obj` creates the shape node and its parent edge.
|
||||
- `:mod-obj` applies supported `:set` operations to graph columns.
|
||||
- `:mod-obj` keeps false and zero values as values.
|
||||
- `:del-obj` deletes shapes in deep post-order.
|
||||
- `:mov-objects` detaches shapes from the old parent.
|
||||
- `:mov-objects` closes the old sibling position gap.
|
||||
- `:mov-objects` inserts shapes at the new position.
|
||||
- `:mov-objects` updates `parent_id` and `frame_id`.
|
||||
- `:mov-objects` rewrites container `shapes` values.
|
||||
- The parent columns and child lists must match a cold projection.
|
||||
|
||||
### Page and component change rules
|
||||
|
||||
- Page add creates a projected page node and a document edge.
|
||||
- Page delete removes the page subtree.
|
||||
- Page modification updates supported page attributes.
|
||||
- Component add creates a component node and document edge.
|
||||
- Component modification updates supported component attributes.
|
||||
- Component delete uses a soft-delete state.
|
||||
- Component restore removes the soft-delete state.
|
||||
- Component purge removes the component node and document edge.
|
||||
- Component sync paths need more parity coverage than the current tests provide.
|
||||
|
||||
## Session Locking
|
||||
|
||||
- The sync loop and HTTP handlers share one lock per session.
|
||||
- The lock protects one Ladybug connection from concurrent access.
|
||||
- Queries acquire the lock before binder validation and execution.
|
||||
- Graph data export acquires the lock before catalog reads.
|
||||
- Session export acquires the lock before `EXPORT DATABASE`.
|
||||
- A long query blocks sync for the same session.
|
||||
- A sync batch blocks queries for the same session.
|
||||
- Ladybug connection thread safety is not assumed.
|
||||
|
||||
## Graph Query Rules
|
||||
|
||||
- The console accepts Cypher text.
|
||||
- Blank query text raises a validation error.
|
||||
- The query first passes Ladybug prepare and bind checks.
|
||||
- The query must pass the engine read-only analysis.
|
||||
- A mutating query is rejected.
|
||||
- The graph console does not provide a write path.
|
||||
- A session graph is rebuilt from the file by Reload.
|
||||
- Normal query results have a 200-row limit.
|
||||
- Query results use string values for the HTML console representation.
|
||||
- JSON requests receive a Transit JSON response with the query and result.
|
||||
- HTML requests receive the rendered console with the result.
|
||||
|
||||
## Graph Data Export
|
||||
|
||||
### G6 data
|
||||
|
||||
- `/dbg/actions/graph-data` reads the live Ladybug database.
|
||||
- It does not read the sync index for nodes and edges.
|
||||
- It therefore shows database drift if a batch fails after index update.
|
||||
- Node export covers all registered node tables.
|
||||
- Relationship export reads the Ladybug relationship catalog.
|
||||
- Relationship export includes source, target, relationship name, and position.
|
||||
- Node and relationship export uses a 100,000-row limit.
|
||||
- The response reports `truncated` when a limit cuts the result.
|
||||
- The response reports buffer-manager memory usage.
|
||||
|
||||
### `.lbug` export
|
||||
|
||||
- `source=file` rebuilds the persistent graph from PostgreSQL file data.
|
||||
- `source=file` runs a synchronous full ingest for each request.
|
||||
- `source=session` exports the caller profile's live in-memory graph.
|
||||
- Session export uses Ladybug `EXPORT DATABASE` to Parquet files.
|
||||
- Session export creates a new `.lbug` database with `IMPORT DATABASE`.
|
||||
- The temporary Parquet staging directory is deleted after import.
|
||||
- The final session `.lbug` file remains in the system temporary directory.
|
||||
- The HTTP response streams the database file to the caller.
|
||||
|
||||
## HTTP Routes and Access
|
||||
|
||||
- The graph routes live in `backend/src/app/http/debug.clj`.
|
||||
- The graph route list is added only when `:graph` is enabled.
|
||||
- `/dbg/graph` serves the graph console page.
|
||||
- `/dbg/actions/graph-files` returns the profile file tree.
|
||||
- `/dbg/actions/graph-load` loads a file into the profile session.
|
||||
- `/dbg/actions/graph-unload` closes the profile session.
|
||||
- `/dbg/actions/graph-reload` rebuilds the loaded file graph.
|
||||
- `/dbg/actions/graph-query` runs a read-only Cypher query.
|
||||
- `/dbg/actions/graph-sync-status` returns the sync state.
|
||||
- `/dbg/actions/graph-data` returns nodes and edges for G6.
|
||||
- `/dbg/actions/graph-export` streams a `.lbug` database.
|
||||
- The `/dbg` session middleware remains active.
|
||||
- The `/dbg` admin middleware remains active.
|
||||
- A devenv host with a profile ID passes the debug authorization rule.
|
||||
- Other hosts need a profile email in the configured admin set.
|
||||
- `/dbg/actions/graph-files` lists reachable teams, projects, and files.
|
||||
- The file tree query has a 500-file limit.
|
||||
- The graph handlers resolve graph namespaces at call time.
|
||||
- The backend requires `app.graph.debug` and `app.graph.ingest` when the flag is on.
|
||||
- Ladybug native loading then fails during route initialization instead of first use.
|
||||
|
||||
## Console Frontend
|
||||
|
||||
### Page type
|
||||
|
||||
- `graph-console.tmpl` is a backend resource template.
|
||||
- It is not a Rumext component.
|
||||
- It is not part of the main frontend route table.
|
||||
- The page uses browser `fetch` calls and a browser WebSocket.
|
||||
- The page loads G6 version `5.1.1` from jsDelivr.
|
||||
|
||||
### File tree
|
||||
|
||||
- The page fetches `/dbg/actions/graph-files`.
|
||||
- The response contains team, project, and file groups.
|
||||
- The page creates the tree with DOM APIs.
|
||||
- A file click submits the graph load form.
|
||||
- The page shows a message when no file exists.
|
||||
|
||||
### Graph rendering
|
||||
|
||||
- The page fetches `/dbg/actions/graph-data`.
|
||||
- The page converts graph nodes and edges to G6 data.
|
||||
- The page skips repaint when the node and edge signature does not change.
|
||||
- The page marks added, removed, and changed graph entities.
|
||||
- The page supports tree, dagre, circular, force, and combo layouts.
|
||||
- The page supports collapsed container combos.
|
||||
- The page has render guards at 4,000 nodes and 8,000 edges.
|
||||
- The `?safe` query option bypasses the render guard.
|
||||
- The page shows graph size by node count and relationship count.
|
||||
- The page shows buffer-manager memory in MiB.
|
||||
- The page reports a CDN failure when G6 is undefined.
|
||||
|
||||
### Query result filtering
|
||||
|
||||
- A query can return `filter_*` columns with node IDs.
|
||||
- The HTML result table hides columns with the `filter_` prefix.
|
||||
- The JSON result keeps the full result.
|
||||
- The graph view uses the hidden IDs to select matching nodes.
|
||||
- The graph view re-runs the query after graph refresh.
|
||||
- This keeps the query filter aligned with the current graph.
|
||||
- A user column named `filter_*` follows the same hiding rule.
|
||||
|
||||
### Node inspector
|
||||
|
||||
- A node click creates a query for that node.
|
||||
- The inspector calls `/dbg/actions/graph-query` with JSON negotiation.
|
||||
- The inspector displays the full projected row.
|
||||
- The inspector uses table and ID values from the graph data.
|
||||
|
||||
## WebSocket Data Flow
|
||||
|
||||
1. The page opens `/ws/notifications` with a random `session-id` query value.
|
||||
2. The page sends `:subscribe-file` with a Transit UUID value.
|
||||
3. The server makes sure that the file exists and that the profile has read permission.
|
||||
4. The server subscribes the connection to the file topic.
|
||||
5. `files_update` publishes `:file-change` to the same topic.
|
||||
6. The graph session consumes the message from its message bus subscription.
|
||||
7. The WebSocket server sends the message to the browser connection.
|
||||
8. The browser adds the change to the changelog.
|
||||
9. The browser fetches sync status after 150 milliseconds.
|
||||
10. The browser fetches graph data after a 400-millisecond debounce.
|
||||
11. The browser repaints the G6 graph when the graph data changes.
|
||||
|
||||
### WebSocket reconnect behavior
|
||||
|
||||
- The page reconnects after three seconds when the socket closes.
|
||||
- The page resubscribes to the file after the socket opens.
|
||||
- The page refreshes sync status after reconnect.
|
||||
- The page refreshes graph data after reconnect.
|
||||
- Reconnect does not recover dropped message-bus changes.
|
||||
- The page shows the sync error or skipped-change state when the status reports it.
|
||||
|
||||
## Feature Flag and Runtime Dependencies
|
||||
|
||||
- `:graph` is defined in `common/src/app/common/flags.cljc`.
|
||||
- The flag is off by default.
|
||||
- `com.ladybugdb/lbug` version `0.19.1` is a backend dependency.
|
||||
- `org.apache.arrow/arrow-memory-netty` version `18.2.0` supports Arrow `RootAllocator`.
|
||||
- The JVM uses `--enable-native-access=ALL-UNNAMED`.
|
||||
- The JVM uses `--add-opens=java.base/java.nio=ALL-UNNAMED`.
|
||||
- The JVM uses `--sun-misc-unsafe-memory-access=allow`.
|
||||
- The JVM options appear in the development alias and backend launch scripts.
|
||||
- A Ladybug version change needs new binder and parity tests.
|
||||
- A JDK version change needs a startup test with the graph flag enabled.
|
||||
|
||||
## Tests
|
||||
|
||||
### `backend-tests.graph-sync-parity-test`
|
||||
|
||||
- Uses two Ladybug `:memory:` databases.
|
||||
- Does not use PostgreSQL or a live graph session.
|
||||
- Projects initial file data into database A.
|
||||
- Applies changes to database A through incremental sync.
|
||||
- Applies the same changes to file data.
|
||||
- Projects the changed file data into database B.
|
||||
- Compares every node row and relationship row.
|
||||
- Reports differences by table, row key, and column.
|
||||
- Covers shape add, shape modification, shape deletion, movement, and page changes.
|
||||
- Contains a test that injects a sync defect and expects a graph difference.
|
||||
- Does not cover all component change variants.
|
||||
- Does not cover every movement insertion mode.
|
||||
|
||||
### `backend-tests.graph-binder-gate-test`
|
||||
|
||||
- Creates the live graph DDL in a Ladybug `:memory:` database.
|
||||
- Prepares each sync statement template without executing it.
|
||||
- Detects parse errors and missing tables.
|
||||
- Detects missing columns and bad label quoting.
|
||||
- Reports the expected read-only classification.
|
||||
- Covers reserved node labels across the node registry.
|
||||
- Reports an error result for an invalid statement.
|
||||
|
||||
### Test gaps
|
||||
|
||||
- No automated HTTP handler tests cover graph routes.
|
||||
- No automated session lifecycle tests cover load and unload.
|
||||
- No automated WebSocket tests cover graph subscription.
|
||||
- No automated export tests cover persistent and session sources.
|
||||
- Component add, modify, delete, restore, and purge need parity tests.
|
||||
- Page delete needs parity coverage.
|
||||
- Movement with `:after-shape` needs parity coverage.
|
||||
- Buffer overflow and revision gap behavior need tests.
|
||||
- Partial batch failure and recovery need tests.
|
||||
- Query timeout and long-query behavior need tests.
|
||||
|
||||
## Known Risks and Limits
|
||||
|
||||
### Dropped changes
|
||||
|
||||
- The sync channel uses a dropping buffer of 64.
|
||||
- A burst can discard file-change messages.
|
||||
- The sync loop logs a revision gap when it sees a larger revision.
|
||||
- The sync loop does not fetch missing rows from `file_change`.
|
||||
- Reload is the only built-in recovery path.
|
||||
|
||||
### Partial batch state
|
||||
|
||||
- `apply-changes!` does not provide Ladybug transaction atomicity.
|
||||
- A statement failure can leave a partly changed graph.
|
||||
- The in-memory index can advance before the database state is complete.
|
||||
- `/dbg/actions/graph-data` reads the database and exposes this drift.
|
||||
- Reload rebuilds the graph from PostgreSQL file data.
|
||||
|
||||
### Query resource use
|
||||
|
||||
- The default session query timeout is zero.
|
||||
- A costly query can hold the session lock for a long time.
|
||||
- The same lock blocks incremental sync.
|
||||
- The graph export also holds the same lock during catalog reads.
|
||||
- The graph schema has a high memory floor.
|
||||
- The console reports about 115 MiB for the wide slice before file data.
|
||||
|
||||
### Session lifecycle
|
||||
|
||||
- Sessions have no TTL.
|
||||
- Sessions remain until unload, replacement, or process shutdown.
|
||||
- Each session owns native Ladybug memory.
|
||||
- Many profiles can create many native databases.
|
||||
- A profile load replaces its previous session.
|
||||
- Two browser tabs for one profile share one graph session.
|
||||
|
||||
### Temporary files
|
||||
|
||||
- Session export leaves the final `.lbug` file in the system temporary directory.
|
||||
- Long-lived servers can accumulate exported session databases.
|
||||
- The staging directory is deleted after import.
|
||||
|
||||
### Browser dependency
|
||||
|
||||
- The graph view depends on a runtime CDN request.
|
||||
- A network restriction can remove the G6 view.
|
||||
- Queries and session status still use backend endpoints without G6.
|
||||
|
||||
### Data exposure
|
||||
|
||||
- The graph console can list many files available to the profile.
|
||||
- The console can load complete projected file data.
|
||||
- The console can export a graph database.
|
||||
- The console can inspect all projected node attributes.
|
||||
- The console is safe only when the `/dbg` access boundary is correct.
|
||||
- The graph flag must remain off for deployments that do not need this tool.
|
||||
|
||||
### Contract drift
|
||||
|
||||
- The graph schema is a deliberate slice of the Penpot file model.
|
||||
- New source attributes do not enter the graph automatically in all cases.
|
||||
- Dropped and unprojected attributes need an explicit contract decision.
|
||||
- `applied_tokens` key mapping depends on the JSON naming function.
|
||||
- `filter_*` is a frontend convention, not a graph schema guarantee.
|
||||
|
||||
### Ladybug dialect coupling
|
||||
|
||||
- Cypher strings contain Ladybug-specific syntax.
|
||||
- Label quoting handles reserved labels explicitly.
|
||||
- Relationship transforms depend on Ladybug relationship limits.
|
||||
- Arrow loading depends on Ladybug `COPY FROM (MATCH ...)` behavior.
|
||||
- A dependency upgrade needs schema, binder, Arrow, and parity checks.
|
||||
|
||||
## REPL Helpers
|
||||
|
||||
- `app.srepl.main` resolves graph functions only when a helper runs.
|
||||
- `graph-smoke-test!` runs a basic Ladybug operation.
|
||||
- `graph-query-test!` runs a graph query test.
|
||||
- `ingest-file-to-graph!` projects a file into a graph database.
|
||||
- These helpers use `requiring-resolve` to keep the graph dependency lazy.
|
||||
|
||||
## Operational Invariants
|
||||
|
||||
- PostgreSQL file data remains authoritative.
|
||||
- Cold projection and incremental sync must produce equal graph state.
|
||||
- The graph revision must identify the last applied file revision.
|
||||
- The document revision must update when a sync batch applies.
|
||||
- A missing or skipped change must remain visible in sync status.
|
||||
- A graph query from the console must be read-only.
|
||||
- A graph session must serialize connection access.
|
||||
- Graph routes must remain behind the `:graph` flag and `/dbg` access control.
|
||||
- The Arrow allocator must outlive all Ladybug operations that use its buffers.
|
||||
- `GraphMeta` must be written after the full ingest and transforms finish.
|
||||
|
||||
## Key Files
|
||||
|
||||
- `backend/src/app/graph/ladybug.clj`: Ladybug API and query gates.
|
||||
- `backend/src/app/graph/arrow.clj`: Arrow bulk load.
|
||||
- `backend/src/app/graph/ingest.clj`: Complete file ingest.
|
||||
- `backend/src/app/graph/debug.clj`: Session lifecycle, sync loop, query, and export.
|
||||
- `backend/src/app/graph/sync.clj`: Incremental change application.
|
||||
- `backend/src/app/graph/meta.clj`: Graph provenance.
|
||||
- `backend/src/app/graph/stats.clj`: Graph counts.
|
||||
- `backend/src/app/graph/report.clj`: REPL ingest report.
|
||||
- `backend/src/app/graph/projection/document.clj`: Base document projection.
|
||||
- `backend/src/app/graph/projection/transforms.clj`: Derived relationship transforms.
|
||||
- `backend/src/app/graph/schema/nodes.clj`: Node and relationship registry.
|
||||
- `backend/src/app/graph/schema/contract.clj`: Projection contract decisions.
|
||||
- `backend/src/app/graph/schema/projection.clj`: Malli projection schemas.
|
||||
- `backend/src/app/graph/schema/types.clj`: Malli-to-Ladybug type mapping.
|
||||
- `backend/src/app/graph/schema/values.clj`: Value coercion.
|
||||
- `backend/src/app/http/debug.clj`: Graph route registration and handlers.
|
||||
- `backend/src/app/http/websocket.clj`: File WebSocket subscription handlers.
|
||||
- `backend/src/app/rpc/commands/files_update.clj`: File-change publication.
|
||||
- `backend/src/app/main.clj`: Integrant message bus wiring.
|
||||
- `backend/resources/app/templates/graph-console.tmpl`: Graph console browser code.
|
||||
- `backend/resources/app/templates/debug.tmpl`: Debug page graph links.
|
||||
- `common/src/app/common/flags.cljc`: `:graph` feature flag.
|
||||
- `backend/test/backend_tests/graph_sync_parity_test.clj`: Cold versus sync parity.
|
||||
- `backend/test/backend_tests/graph_binder_gate_test.clj`: Cypher binder gate.
|
||||
|
||||
## Development Commands
|
||||
|
||||
- Run backend commands from the `backend/` directory.
|
||||
- Run focused parity tests with `clojure -M:dev:test --focus backend-tests.graph-sync-parity-test`.
|
||||
- Run focused binder tests with `clojure -M:dev:test --focus backend-tests.graph-binder-gate-test`.
|
||||
- Run the backend test suite with `clojure -M:dev:test`.
|
||||
- Examine Clojure formatting with `pnpm run check-fmt:clj`.
|
||||
- Run backend Clojure lint with `pnpm run lint:clj`.
|
||||
- Write test output to a file before reading or filtering it.
|
||||
@@ -6,6 +6,7 @@ Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Par
|
||||
|
||||
- `penpotdev-infra`: shared `postgres`, `minio`, `minio-setup`, `mailer`, `ldap`. File: `docker-compose.infra.yml`.
|
||||
- `penpotdev-wsN` (N=0,1,…): per-instance `main` + `redis` (Valkey). File: `docker-compose.main.yml`. ws0 (a.k.a. `main`) binds `$PWD`; ws1+ bind clones at `${PENPOT_WORKSPACES_DIR}/wsN/` (default `~/.penpot/penpot_workspaces/`), maintained by the developer.
|
||||
- Optional overlay `docker-compose.opencode.yml`: added by `instance-compose` as an extra `-f` only when `PENPOT_OPENCODE_CONFIG_DIR` is set (i.e. `run-devenv --opencode-config-dir DIR` ran in this process). Bind-mounts the host dir at `/home/penpot/.config/opencode` (`:z`). Flag-only, per-call; not read from ambient env. Parser `parse-opencode-config-dir` absolutizes (`~`, realpath) because compose resolves relative bind sources against the compose file's dir. Only instances brought up with the flag get the mount.
|
||||
- All projects join external network `penpot_shared`. Created idempotently by `ensure-devenv-network`, never removed by lifecycle commands.
|
||||
|
||||
## Source-of-truth files
|
||||
@@ -65,7 +66,7 @@ No `--delete` on the working-tree pass: gitignored caches in the workspace survi
|
||||
|
||||
## CLI surface
|
||||
|
||||
- `run-devenv --agentic [--ws main|0|wsN|N] [--sync] [--serena-context CTX]`: bring one instance up. Agentic only — MCP and Serena windows are always created. Default target main. Errors out if the target is already running. `--sync` is rejected on main; on ws1+ it's optional (forced only when the workspace dir does not exist yet).
|
||||
- `run-devenv --agentic [--ws main|0|wsN|N] [--sync] [--serena-context CTX] [--opencode-config-dir DIR]`: bring one instance up. Agentic only — MCP and Serena windows are always created. Default target main. Errors out if the target is already running. `--sync` is rejected on main; on ws1+ it's optional (forced only when the workspace dir does not exist yet). `--opencode-config-dir DIR` bind-mounts DIR at `~/.config/opencode` in-container via the optional overlay above; mount applies at container creation, so changing it requires stop + re-run.
|
||||
- `stop-devenv [--ws main|0|wsN|N] [--all]`: stop instances. Flags mutually exclusive. `--ws N` stops just that workspace. `--ws 0` or no flag stops ws0; shared infra shuts down only if no other instances remain. `--all` stops every ws highest-first then ws0, then infra.
|
||||
- `run-devenv`: legacy alias, ws0 non-agentic attached.
|
||||
- `attach-devenv [--ws main|0|wsN|N]`: pure attach. Fails fast if instance/session missing.
|
||||
|
||||
@@ -19,6 +19,20 @@
|
||||
|
||||
- Raster `Fill::Image`: skip `save_layer` unless the shape has an image filter; plain
|
||||
Rect/Frame (no corners) also skip the container clip (`draw_image_fill` in fills.rs).
|
||||
- `can_render_directly` paints onto Current (no Fills/Strokes blit) for plain geometry and
|
||||
for stroke-free text (SrcOver, no blur/shadows). Multi-style text is fine: span styles
|
||||
live in Paragraph `TextStyle`s. Text skips the `nested_fills` guard (fills are on spans).
|
||||
`draw_text` only `save_layer`s when stroke-group opacity is set; plain fill paint is direct.
|
||||
- Plain text fill paint reuses `TextContent.layout` paragraphs when
|
||||
`has_usable_paint_layout` (paragraphs present + version match; during
|
||||
interactive transforms rotation/move skips width check via
|
||||
`modifier_changes_text_layout`, resize falls back to `layout_width` vs
|
||||
`get_width(selrect.width())`), via `text::try_paint_from_layout_cache`.
|
||||
The walker computes `text_layout_cache_rotation_only` from `tree` and
|
||||
passes it into `render_shape`; stroke/shadow paths pass `false`.
|
||||
- `TextContentLayout` paragraphs are `Rc`-shared on `Clone` so modifier clones
|
||||
(rotate/pan) keep the paint cache; `needs_update` is paragraphs-empty only.
|
||||
Decorations are skipped when no span requests underline/strike.
|
||||
- Zoom settle: visible tiles present via `FrameType::ViewportReady` before interest-ring
|
||||
work; crop-cache rebuild is deferred to the later `Full` so the soft→sharp snap is
|
||||
compose+present only.
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
# Creating Pull Requests
|
||||
|
||||
PR only on explicit request. Branch: issue/feature-specific; fallback `<type>/<short-description>` (`fix/...`, `feat/...`, `refactor/...`, `docs/...`, `chore/...`, `perf/...`).
|
||||
PR only on explicit request.
|
||||
|
||||
## Branch Naming
|
||||
|
||||
- Primary: `issue-NNNN` — one branch per GitHub issue (e.g. `issue-11525`).
|
||||
- No issue: free-form descriptive name, dash-separated, no slashes (e.g. `fix-ellipse-icon-typo`, `feat-auto-link-libraries`).
|
||||
- If the user already created the branch, use it as-is — never rename.
|
||||
|
||||
## Target Branch
|
||||
|
||||
|
||||
@@ -40,10 +40,27 @@ Skipping this step is the #1 cause of incorrect or incomplete work.
|
||||
`github.com/penpot/penpot/security/advisories/GHSA-*`, extract the GHSA ID
|
||||
from the URL and run `python3 scripts/gh.py advisories <GHSA-ID>` to fetch
|
||||
full advisory details before proceeding.
|
||||
- **Issue or PR mentioned** — When the user mentions a penpot/penpot issue or
|
||||
PR (URL like `github.com/penpot/penpot/issues/<n>` / `.../pull/<n>`, or a
|
||||
bare `#<n>` when context clearly refers to this repo), fetch details via CLI
|
||||
instead of WebFetch:
|
||||
- Issue → `gh issue view <n> --repo penpot/penpot` (add `--comments` when
|
||||
discussion context matters).
|
||||
- Single PR → `gh pr view <n> --repo penpot/penpot`.
|
||||
- Multiple PRs (list, file, or milestone) → `python3 scripts/gh.py prs ...`.
|
||||
Do this before proceeding. Only use WebFetch if the CLI fails.
|
||||
|
||||
## Writing Rules
|
||||
|
||||
Use the `ste` skill when the user explicitly requests STE, `/ste`, or ASD-STE100.
|
||||
Writing rules, from Orwell, 1946. These govern prose: docs, PR text, messages. Never touch code or technical terms; swap in everyday words only where precision survives.
|
||||
|
||||
1. Never use a metaphor, simile or other figure of speech which you are used to seeing in print.
|
||||
2. Never use a long word where a short one will do.
|
||||
3. If it is possible to cut a word out, always cut it out.
|
||||
4. Never use the passive where you can use the active.
|
||||
5. Never use a foreign phrase, a scientific word or a jargon word if you can think of an everyday English equivalent.
|
||||
6. Break any of these rules sooner than say anything outright barbarous.
|
||||
Review every prose output against these rules before delivering.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+127
-4
@@ -1,5 +1,25 @@
|
||||
# CHANGELOG
|
||||
|
||||
## 2.19.0 (Unreleased)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix copying text from Penpot to the clipboard not working on MS Windows [#11303](https://github.com/penpot/penpot/issues/11303) (PR: [#11305](https://github.com/penpot/penpot/pull/11305))
|
||||
- Fix performance issue with WebGL render [#11240](https://github.com/penpot/penpot/issues/11240) (PR: [#11259](https://github.com/penpot/penpot/pull/11259))
|
||||
- Fix comment bubbles rendering on top of workspace dropdown menus [#10283](https://github.com/penpot/penpot/issues/10283) (PR: [#11201](https://github.com/penpot/penpot/pull/11201))
|
||||
- Fix inconsistent Mixed label in blur options and numeric inputs across 24 locales (by @filipsajdak) [#11148](https://github.com/penpot/penpot/issues/11148) (PR: [#11151](https://github.com/penpot/penpot/pull/11151))
|
||||
- Fix overlay shifting left when shown with top-center alignment in viewer prototype (by @filipsajdak) [#9048](https://github.com/penpot/penpot/issues/9048) (PR: [#10454](https://github.com/penpot/penpot/pull/10454))
|
||||
- Fix internal error when clicking the Copy button on the Access Token page (by @0xTHAC0) [#8496](https://github.com/penpot/penpot/issues/8496) (PR: [#11156](https://github.com/penpot/penpot/pull/11156))
|
||||
- Fix `disable-registration` flag not preventing non-users from creating accounts in the share prototypes page (by @0xTHAC0) [#5164](https://github.com/penpot/penpot/issues/5164) (PR: [#11199](https://github.com/penpot/penpot/pull/11199))
|
||||
|
||||
### :sparkles: New features & Enhancements
|
||||
|
||||
- Make backend storage resilient to interrupted writes, missing files and stalled cleanup [#11344](https://github.com/penpot/penpot/issues/11344) (PR: [#11345](https://github.com/penpot/penpot/pull/11345))
|
||||
- Implement RTL support in the text editor v3 [#11262](https://github.com/penpot/penpot/issues/11262)
|
||||
- Improve path operations and edition in the path editor [#10889](https://github.com/penpot/penpot/issues/10889) (PR: [#10807](https://github.com/penpot/penpot/pull/10807))
|
||||
- Add configurable keyboard shortcuts [#9924](https://github.com/penpot/penpot/issues/9924) (PR: [#10237](https://github.com/penpot/penpot/pull/10237))
|
||||
- Add auto-linking of libraries during import based on slugified name [#9263](https://github.com/penpot/penpot/issues/9263) (PR: [#9958](https://github.com/penpot/penpot/pull/9958))
|
||||
|
||||
## 2.18.0 (Unreleased)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
@@ -11,18 +31,121 @@
|
||||
- Fix plugin API addTheme calls failing with the signature shown in the high-level overview [#10074](https://github.com/penpot/penpot/issues/10074) (PR: [#10359](https://github.com/penpot/penpot/pull/10359))
|
||||
- Fix empty text shape not being deleted on editor exit [#10540](https://github.com/penpot/penpot/issues/10540) (PR: [#10541](https://github.com/penpot/penpot/pull/10541))
|
||||
- Fix broken token pills showing wrong default state when not selected [#10524](https://github.com/penpot/penpot/issues/10524) (PR: [#10535](https://github.com/penpot/penpot/pull/10535))
|
||||
- Replace hyphens with bullets in subscription benefits list [#10547](https://github.com/penpot/penpot/issues/10547) (PR: [#10523](https://github.com/penpot/penpot/pull/10523))
|
||||
- Fix Chinese (zh-CN) translation showing wrong label for Intersection in board path menu (by @sawirricardo) [#10346](https://github.com/penpot/penpot/issues/10346) (PR: [#10381](https://github.com/penpot/penpot/pull/10381))
|
||||
- Fix invalid formulas being accepted in numeric inputs (by @AKnassa) [#9581](https://github.com/penpot/penpot/issues/9581) (PR: [#10659](https://github.com/penpot/penpot/pull/10659))
|
||||
- Fix radial gradient handles blowing up in size when rotated on ellipses (by @AKnassa) [#10069](https://github.com/penpot/penpot/issues/10069) (PR: [#10666](https://github.com/penpot/penpot/pull/10666))
|
||||
- Fix plugin API validation errors being too generic to diagnose the failure (by @AKnassa) [#10072](https://github.com/penpot/penpot/issues/10072) (PR: [#10667](https://github.com/penpot/penpot/pull/10667))
|
||||
- Fix crash with referential integrity error when deleting a component inside a grid (by @Alotor) [#10101](https://github.com/penpot/penpot/issues/10101) (PR: [#10956](https://github.com/penpot/penpot/pull/10956))
|
||||
- Fix component copies not preserving rotation when the main component has changes [#10109](https://github.com/penpot/penpot/issues/10109) (PR: [#10574](https://github.com/penpot/penpot/pull/10574))
|
||||
- Fix text width and height staying stale after setting growType in the plugin API [#10207](https://github.com/penpot/penpot/issues/10207) (PR: [#9898](https://github.com/penpot/penpot/pull/9898))
|
||||
- Fix padding not painted until expanding the 4-sides padding option [#10278](https://github.com/penpot/penpot/issues/10278) (PR: [#10602](https://github.com/penpot/penpot/pull/10602))
|
||||
- Fix files with custom fonts breaking with a referential integrity error when moved between teams (by @filipsajdak) [#10496](https://github.com/penpot/penpot/issues/10496) (PR: [#10837](https://github.com/penpot/penpot/pull/10837))
|
||||
- Fix clicking overlapping comment bubbles zooming to 20000% without showing the comments [#10526](https://github.com/penpot/penpot/issues/10526) (PR: [#10543](https://github.com/penpot/penpot/pull/10543))
|
||||
- Fix user menu subsections in the dashboard not closing when hovering away from the parent option (by @AKnassa) [#10549](https://github.com/penpot/penpot/issues/10549) (PR: [#10639](https://github.com/penpot/penpot/pull/10639))
|
||||
- Fix self-hosted env-generated config.js being cached for 7 days so PENPOT_FLAGS changes did not reach already-cached browsers (by @filipsajdak) [#10556](https://github.com/penpot/penpot/issues/10556) (PR: [#11146](https://github.com/penpot/penpot/pull/11146))
|
||||
- Fix color of selected text in light theme [#10570](https://github.com/penpot/penpot/issues/10570) (PR: [#10614](https://github.com/penpot/penpot/pull/10614))
|
||||
- Fix margin input order being inconsistent with padding inputs and between collapsed and expanded states [#10578](https://github.com/penpot/penpot/issues/10578) (PR: [#10797](https://github.com/penpot/penpot/pull/10797))
|
||||
- Fix uncaught DOMException when writing image/svg+xml content to the clipboard (by @AKnassa) [#10596](https://github.com/penpot/penpot/issues/10596) (PR: [#10663](https://github.com/penpot/penpot/pull/10663))
|
||||
- Fix tick icons not aligned in the font selector [#10597](https://github.com/penpot/penpot/issues/10597) (PR: [#10774](https://github.com/penpot/penpot/pull/10774))
|
||||
- Fix incorrect padding values when multiple shapes are selected [#10598](https://github.com/penpot/penpot/issues/10598) (PR: [#10602](https://github.com/penpot/penpot/pull/10602))
|
||||
- Fix integrity errors related to variants not being repaired [#10606](https://github.com/penpot/penpot/issues/10606) (PR: [#10768](https://github.com/penpot/penpot/pull/10768))
|
||||
- Fix changing password showing 'Password should be at least 8 characters' error on the old password field (by @AKnassa) [#10626](https://github.com/penpot/penpot/issues/10626) (PR: [#10661](https://github.com/penpot/penpot/pull/10661))
|
||||
- Fix stroke caps disappearing when dragging [#10633](https://github.com/penpot/penpot/issues/10633) (PR: [#10634](https://github.com/penpot/penpot/pull/10634))
|
||||
- Fix layout padding being saved as string after invalid input in multi-selection, causing persistence errors (by @niwinz) [#10638](https://github.com/penpot/penpot/issues/10638) (PR: [#10758](https://github.com/penpot/penpot/pull/10758))
|
||||
- Fix inconsistent theme handling between Penpot and plugins [#10676](https://github.com/penpot/penpot/issues/10676) (PR: [#10677](https://github.com/penpot/penpot/pull/10677))
|
||||
- Fix image stroke (strokeImage) support missing in the plugin API Stroke interface [#10682](https://github.com/penpot/penpot/issues/10682) (PR: [#10683](https://github.com/penpot/penpot/pull/10683))
|
||||
- Fix SVG images not working as fill in the WebGL renderer [#10705](https://github.com/penpot/penpot/issues/10705) (PR: [#10707](https://github.com/penpot/penpot/pull/10707))
|
||||
- Fix background blur not working on text shapes [#10706](https://github.com/penpot/penpot/issues/10706) (PR: [#10712](https://github.com/penpot/penpot/pull/10712))
|
||||
- Fix background blur not applying on strokes [#10713](https://github.com/penpot/penpot/issues/10713) (PR: [#10716](https://github.com/penpot/penpot/pull/10716))
|
||||
- Fix text shape with empty content breaking workspace updates [#10725](https://github.com/penpot/penpot/issues/10725) (PR: [#10731](https://github.com/penpot/penpot/pull/10731))
|
||||
- Fix missing SVG option in the file filters when adding an image fill (by @LuBoys) [#10756](https://github.com/penpot/penpot/issues/10756) (PR: [#10771](https://github.com/penpot/penpot/pull/10771))
|
||||
- Update onboarding image [#10779](https://github.com/penpot/penpot/issues/10779) (PR: [#10783](https://github.com/penpot/penpot/pull/10783))
|
||||
- Fix main toolbar overlapping the grid edition bar [#10788](https://github.com/penpot/penpot/issues/10788) (PR: [#10789](https://github.com/penpot/penpot/pull/10789))
|
||||
- Fix WASM renderer panic when the WebGL context is restored mid-reload [#10810](https://github.com/penpot/penpot/issues/10810) (PR: [#10824](https://github.com/penpot/penpot/pull/10824))
|
||||
- Fix nginx frontend forwarding the client Host header to backend/exporter, breaking Istio strict mTLS routing (by @yamila-moreno) [#10835](https://github.com/penpot/penpot/issues/10835) (PR: [#11233](https://github.com/penpot/penpot/pull/11233))
|
||||
- Fix tutorial templates with components causing errors [#10839](https://github.com/penpot/penpot/issues/10839)
|
||||
- Fix plugin 'Try out' flow crashing when projects have not loaded yet [#10858](https://github.com/penpot/penpot/issues/10858) (PR: [#10859](https://github.com/penpot/penpot/pull/10859))
|
||||
- Fix collapsed Fill color section on the design panel for new texts [#10860](https://github.com/penpot/penpot/issues/10860) (PR: [#10972](https://github.com/penpot/penpot/pull/10972))
|
||||
- Fix grid item date tooltip in the project view showing 'Will be deleted' instead of creation date (by @0xTHAC0) [#10873](https://github.com/penpot/penpot/issues/10873) (PR: [#11161](https://github.com/penpot/penpot/pull/11161))
|
||||
- Merge stop and start measurement shortcut to match current behavior [#10884](https://github.com/penpot/penpot/issues/10884) (PR: [#10906](https://github.com/penpot/penpot/pull/10906))
|
||||
- Fix shape size badge displayed twice when a user with Viewer permissions selects a shape [#10893](https://github.com/penpot/penpot/issues/10893) (PR: [#10985](https://github.com/penpot/penpot/pull/10985))
|
||||
- Fix main menu being covered by the toolbar [#10902](https://github.com/penpot/penpot/issues/10902) (PR: [#10926](https://github.com/penpot/penpot/pull/10926))
|
||||
- Fix font family typography asset persisting across files in newly created text layers [#10925](https://github.com/penpot/penpot/issues/10925) (PR: [#11134](https://github.com/penpot/penpot/pull/11134))
|
||||
- Fix error raised when editing justified text [#10944](https://github.com/penpot/penpot/issues/10944) (PR: [#10945](https://github.com/penpot/penpot/pull/10945))
|
||||
- Fix MCP WebSocket proxy failing after penpot-mcp container restarts due to stale nginx DNS resolution (by @780Farva) [#10946](https://github.com/penpot/penpot/issues/10946) (PR: [#10947](https://github.com/penpot/penpot/pull/10947))
|
||||
- Fix verification email address being unreadable due to low-contrast text on the register success page [#10950](https://github.com/penpot/penpot/issues/10950) (PR: [#10965](https://github.com/penpot/penpot/pull/10965))
|
||||
- Fix image swatches displaying a wrong format in the color picker list view [#10951](https://github.com/penpot/penpot/issues/10951) (PR: [#10975](https://github.com/penpot/penpot/pull/10975))
|
||||
- Fix text editor crashing when dropping dragged text after selecting all content [#10954](https://github.com/penpot/penpot/issues/10954) (PR: [#10959](https://github.com/penpot/penpot/pull/10959))
|
||||
- Fix MCP tokens being usable as API access tokens [#10960](https://github.com/penpot/penpot/issues/10960) (PR: [#10962](https://github.com/penpot/penpot/pull/10962))
|
||||
- Add size limit and rate limiting to the send-user-feedback endpoint [#10979](https://github.com/penpot/penpot/issues/10979) (PR: [#10990](https://github.com/penpot/penpot/pull/10990))
|
||||
- Fix main menu not keeping alignment when the left sidebar is expanded [#10981](https://github.com/penpot/penpot/issues/10981) (PR: [#10986](https://github.com/penpot/penpot/pull/10986))
|
||||
- Fix update-profile-props RPC method accepting undocumented keys [#10991](https://github.com/penpot/penpot/issues/10991) (PR: [#10992](https://github.com/penpot/penpot/pull/10992))
|
||||
- Fix import-binfile RPC method schema accepting a file-id parameter [#10993](https://github.com/penpot/penpot/issues/10993) (PR: [#10994](https://github.com/penpot/penpot/pull/10994))
|
||||
- Fix assemble-chunks session lookup ignoring the profile-id scope [#11011](https://github.com/penpot/penpot/issues/11011) (PR: [#11012](https://github.com/penpot/penpot/pull/11012))
|
||||
- Validate font-id team ownership in create-font-variant [#11013](https://github.com/penpot/penpot/issues/11013) (PR: [#11014](https://github.com/penpot/penpot/pull/11014))
|
||||
- Validate team ownership on file library link endpoints [#11015](https://github.com/penpot/penpot/issues/11015) (PR: [#11016](https://github.com/penpot/penpot/pull/11016))
|
||||
- Limit object size allocation in the V1 binfile parser [#11017](https://github.com/penpot/penpot/issues/11017) (PR: [#11018](https://github.com/penpot/penpot/pull/11018))
|
||||
- Limit recursion depth in the Fressian reader [#11019](https://github.com/penpot/penpot/issues/11019) (PR: [#11020](https://github.com/penpot/penpot/pull/11020))
|
||||
- Limit concurrent imports in the import-binfile RPC method [#11023](https://github.com/penpot/penpot/issues/11023) (PR: [#11024](https://github.com/penpot/penpot/pull/11024))
|
||||
- Validate content-type on management upload endpoints [#11025](https://github.com/penpot/penpot/issues/11025) (PR: [#11026](https://github.com/penpot/penpot/pull/11026))
|
||||
- Fix webhook endpoints allowing unauthorized access via creator-id fallback [#11028](https://github.com/penpot/penpot/issues/11028) (PR: [#11029](https://github.com/penpot/penpot/pull/11029))
|
||||
- Escape markdown in user-controlled fields of Mattermost error notifications [#11033](https://github.com/penpot/penpot/issues/11033) (PR: [#11034](https://github.com/penpot/penpot/pull/11034))
|
||||
- Enforce file read permission check on asset endpoints [#11035](https://github.com/penpot/penpot/issues/11035) (PR: [#11036](https://github.com/penpot/penpot/pull/11036))
|
||||
- Add accumulated storage byte quota for media uploads [#11037](https://github.com/penpot/penpot/issues/11037) (PR: [#11038](https://github.com/penpot/penpot/pull/11038))
|
||||
- Add bounding box dimension limit to exports [#11041](https://github.com/penpot/penpot/issues/11041) (PR: [#11042](https://github.com/penpot/penpot/pull/11042))
|
||||
- Sanitize embedded scripts in SVG uploads [#11043](https://github.com/penpot/penpot/issues/11043) (PR: [#11044](https://github.com/penpot/penpot/pull/11044))
|
||||
- Fix duplicate file ID returning inconsistent error responses [#11045](https://github.com/penpot/penpot/issues/11045) (PR: [#11050](https://github.com/penpot/penpot/pull/11050))
|
||||
- Enforce permission checks in WebSocket subscription handlers [#11052](https://github.com/penpot/penpot/issues/11052) (PR: [#11054](https://github.com/penpot/penpot/pull/11054))
|
||||
- Fix 'something went wrong' popup when using incremental numerical input interaction [#11053](https://github.com/penpot/penpot/issues/11053) (PR: [#10794](https://github.com/penpot/penpot/pull/10794))
|
||||
- Enforce password complexity validation on the backend [#11055](https://github.com/penpot/penpot/issues/11055) (PR: [#11059](https://github.com/penpot/penpot/pull/11059))
|
||||
- Normalize string inputs before processing [#11060](https://github.com/penpot/penpot/issues/11060) (PR: [#11061](https://github.com/penpot/penpot/pull/11061))
|
||||
- Add cooldown to avoid sending duplicate invitation emails [#11062](https://github.com/penpot/penpot/issues/11062) (PR: [#11063](https://github.com/penpot/penpot/pull/11063))
|
||||
- Enable SSRF protection for organization SSO validation [#11064](https://github.com/penpot/penpot/issues/11064) (PR: [#11065](https://github.com/penpot/penpot/pull/11065))
|
||||
- Fix clone-file-media-object allowing to clone media objects from files without read access [#11087](https://github.com/penpot/penpot/issues/11087) (PR: [#11090](https://github.com/penpot/penpot/pull/11090))
|
||||
- Fix 404 error page logo not visible in dark mode [#11091](https://github.com/penpot/penpot/issues/11091) (PR: [#11167](https://github.com/penpot/penpot/pull/11167))
|
||||
- Fix incorrect permission handling when creating an invitation [#11098](https://github.com/penpot/penpot/issues/11098) (PR: [#11099](https://github.com/penpot/penpot/pull/11099))
|
||||
- Reject zero or negative total-chunks values in upload sessions [#11103](https://github.com/penpot/penpot/issues/11103) (PR: [#11104](https://github.com/penpot/penpot/pull/11104))
|
||||
- Fix import-binfile accepting unsupported version values without validation [#11105](https://github.com/penpot/penpot/issues/11105) (PR: [#11107](https://github.com/penpot/penpot/pull/11107))
|
||||
- Fix sessions remaining active on other devices after account deletion [#11114](https://github.com/penpot/penpot/issues/11114) (PR: [#11115](https://github.com/penpot/penpot/pull/11115))
|
||||
- Use random UUIDs for share link IDs instead of a predictable scheme [#11116](https://github.com/penpot/penpot/issues/11116) (PR: [#11117](https://github.com/penpot/penpot/pull/11117))
|
||||
- Fix plugin manifest fetch hanging indefinitely without timeout [#11119](https://github.com/penpot/penpot/issues/11119) (PR: [#11120](https://github.com/penpot/penpot/pull/11120))
|
||||
- Use constant-time comparison for shared key authentication [#11121](https://github.com/penpot/penpot/issues/11121) (PR: [#11122](https://github.com/penpot/penpot/pull/11122))
|
||||
- Fix ESC key not closing the comment input box after posting a comment in the workspace [#11128](https://github.com/penpot/penpot/issues/11128) (PR: [#11131](https://github.com/penpot/penpot/pull/11131))
|
||||
- Fix token edit modal crashing when resolving tokens with group nodes [#11143](https://github.com/penpot/penpot/issues/11143) (PR: [#11144](https://github.com/penpot/penpot/pull/11144))
|
||||
- Fix text editor crashing when pasting into an empty text shape [#11149](https://github.com/penpot/penpot/issues/11149) (PR: [#11150](https://github.com/penpot/penpot/pull/11150))
|
||||
- Fix comment avatars appearing on top of rulers when scrolling the canvas (by @filipsajdak) [#11163](https://github.com/penpot/penpot/issues/11163) (PR: [#11168](https://github.com/penpot/penpot/pull/11168))
|
||||
- Fix infinite loop of get-teams and get-team-members calls when granting team access from an email link [#11215](https://github.com/penpot/penpot/issues/11215) (PR: [#11223](https://github.com/penpot/penpot/pull/11223))
|
||||
- Fix RPC requests bypassing rate limiting with fractional bucket refill intervals [#11253](https://github.com/penpot/penpot/issues/11253) (PR: [#11254](https://github.com/penpot/penpot/pull/11254))
|
||||
- Fix tempfile bucket serving objects to any authenticated user instead of only the uploader [#11269](https://github.com/penpot/penpot/issues/11269) (PR: [#11270](https://github.com/penpot/penpot/pull/11270))
|
||||
- Fix increasing a value by clicking and dragging in a numeric input [#11274](https://github.com/penpot/penpot/issues/11274) (PR: [#11334](https://github.com/penpot/penpot/pull/11334))
|
||||
- Fix notification pill rendering unescaped HTML in the detail section when importing tokens [#11276](https://github.com/penpot/penpot/issues/11276) (PR: [#11275](https://github.com/penpot/penpot/pull/11275))
|
||||
- Fix share-link holders reading pages outside the authorized scope via the get-page RPC command [#11281](https://github.com/penpot/penpot/issues/11281) (PR: [#11284](https://github.com/penpot/penpot/pull/11284))
|
||||
- Fix incorrect permission handling when managing share links on a file [#11289](https://github.com/penpot/penpot/issues/11289) (PR: [#11290](https://github.com/penpot/penpot/pull/11290))
|
||||
- Fix backend session remaining valid after logout when the auth-token cookie is replayed [#11316](https://github.com/penpot/penpot/issues/11316) (PR: [#11317](https://github.com/penpot/penpot/pull/11317))
|
||||
- Fix get-team-invitation-token requiring only read permissions [#11358](https://github.com/penpot/penpot/issues/11358) (PR: [#11359](https://github.com/penpot/penpot/pull/11359))
|
||||
|
||||
### :sparkles: New features & Enhancements
|
||||
|
||||
- Group toolbar drawing tools into shape and free-draw flyouts [#9316](https://github.com/penpot/penpot/issues/9316) (PR: [#9480](https://github.com/penpot/penpot/pull/9480), [#10354](https://github.com/penpot/penpot/pull/10354))
|
||||
- Add outline stroke to Paths [#9961](https://github.com/penpot/penpot/issues/9961) (PR: [#8677](https://github.com/penpot/penpot/pull/8677))
|
||||
- Make throwValidationErrors default to true for v2 manifest plugins [#10401](https://github.com/penpot/penpot/issues/10401) (PR: [#10433](https://github.com/penpot/penpot/pull/10433))
|
||||
- Add dedicated Line and Arrow drawing tools (by @davidv399) [#9145](https://github.com/penpot/penpot/issues/9145) (PR: [#9146](https://github.com/penpot/penpot/pull/9146))
|
||||
- Refactor wasm rulers and UI state [#10116](https://github.com/penpot/penpot/issues/10116) (PR: [#10461](https://github.com/penpot/penpot/pull/10461))
|
||||
- Improve team invitations modal in the dashboard [#10484](https://github.com/penpot/penpot/issues/10484) (PR: [#10459](https://github.com/penpot/penpot/pull/10459))
|
||||
|
||||
- Highlight the first matching font in the font list when searching (by @ai-mountain) [#3204](https://github.com/penpot/penpot/issues/3204) (PR: [#9512](https://github.com/penpot/penpot/pull/9512), [#10450](https://github.com/penpot/penpot/pull/10450))
|
||||
- Preserve token references when copying and pasting properties instead of resolving them to values (by @AKnassa) [#9582](https://github.com/penpot/penpot/issues/9582) (PR: [#10665](https://github.com/penpot/penpot/pull/10665))
|
||||
- Add waitForLayoutUpdate method to the plugin API [#10136](https://github.com/penpot/penpot/issues/10136) (PR: [#9898](https://github.com/penpot/penpot/pull/9898))
|
||||
- Show and manage comments while designing in the workspace [#10239](https://github.com/penpot/penpot/issues/10239) (PR: [#10275](https://github.com/penpot/penpot/pull/10275))
|
||||
- Simplify MCP server configuration for common MCP clients [#10355](https://github.com/penpot/penpot/issues/10355) (PR: [#10604](https://github.com/penpot/penpot/pull/10604))
|
||||
- Remove misleading MCP client JSON snippet from the key-generated modal (by @Shlok1729) [#10399](https://github.com/penpot/penpot/issues/10399) (PR: [#10415](https://github.com/penpot/penpot/pull/10415))
|
||||
- Preview font families in the font selector [#10403](https://github.com/penpot/penpot/issues/10403) (PR: [#10411](https://github.com/penpot/penpot/pull/10411))
|
||||
- Remember expanded/collapsed state of token sets in the color tokens picker (session scope) [#10551](https://github.com/penpot/penpot/issues/10551) (PR: [#10864](https://github.com/penpot/penpot/pull/10864))
|
||||
- Show token sets in reverse order by default in the color tokens picker (by @rhinocap) [#10552](https://github.com/penpot/penpot/issues/10552) (PR: [#10658](https://github.com/penpot/penpot/pull/10658))
|
||||
- Add multi-selection and bulk delete support to pages in the workspace sitemap [#10580](https://github.com/penpot/penpot/issues/10580) (PR: [#10581](https://github.com/penpot/penpot/pull/10581))
|
||||
- Add a grid/list view toggle for files in the dashboard [#10691](https://github.com/penpot/penpot/issues/10691) (PR: [#10692](https://github.com/penpot/penpot/pull/10692))
|
||||
- Migrate Docker images to Docker Hardened Images (DHI) [#10720](https://github.com/penpot/penpot/issues/10720) (PR: [#10732](https://github.com/penpot/penpot/pull/10732), [#10733](https://github.com/penpot/penpot/pull/10733), [#10734](https://github.com/penpot/penpot/pull/10734))
|
||||
- Adopt React Aria [#10802](https://github.com/penpot/penpot/issues/10802) (PR: [#10675](https://github.com/penpot/penpot/pull/10675))
|
||||
- Add plugin API function for awaiting component updates beyond waitForLayoutUpdate [#10927](https://github.com/penpot/penpot/issues/10927) (PR: [#10964](https://github.com/penpot/penpot/pull/10964))
|
||||
- Emit open-workspace-file audit event with file statistics on workspace load [#11106](https://github.com/penpot/penpot/issues/11106) (PR: [#11138](https://github.com/penpot/penpot/pull/11138))
|
||||
## 2.17.2
|
||||
|
||||
### :bug: Bugs fixed
|
||||
@@ -3033,7 +3156,7 @@ is a number of cores)
|
||||
- Enable penpot SVG metadata only when exporting complete files [Taiga #1914](https://tree.taiga.io/project/penpot/us/1914?milestone=295883)
|
||||
- Export to PDF all artboards of one page [Taiga #1895](https://tree.taiga.io/project/penpot/us/1895)
|
||||
- Go to a undo step clicking on a history element of the list [Taiga #1374](https://tree.taiga.io/project/penpot/us/1374)
|
||||
- Increment font size by 10 with shift+arrows [1047](https://github.com/penpot/penpot/issues/1047)
|
||||
- Increment font size by 10 with shift+arrows [#1047](https://github.com/penpot/penpot/issues/1047)
|
||||
- New shortcut to detach components Ctrl+Shift+K [Taiga #1799](https://tree.taiga.io/project/penpot/us/1799)
|
||||
- Set email inputs to type "email", to aid keyboard entry [Taiga #1921](https://tree.taiga.io/project/penpot/issue/1921)
|
||||
- Use shift+move to move element orthogonally [#823](https://github.com/penpot/penpot/issues/823)
|
||||
|
||||
@@ -14,6 +14,7 @@ Center](https://help.penpot.app/).
|
||||
- [Reporting Bugs](#reporting-bugs)
|
||||
- [Pull Requests](#pull-requests)
|
||||
- [Workflow](#workflow)
|
||||
- [Branch naming](#branch-naming)
|
||||
- [Format](#format)
|
||||
- [Title format](#title-format)
|
||||
- [Description](#description)
|
||||
@@ -73,6 +74,18 @@ Advisories](https://github.com/penpot/penpot/security/advisories)
|
||||
4. **Format and lint** — run the checks described in
|
||||
[Formatting and Linting](#formatting-and-linting) before submitting.
|
||||
|
||||
### Branch naming
|
||||
|
||||
Branch names are not enforced, but we recommend the following:
|
||||
|
||||
- **`issue-NNNN`** — when working from a GitHub issue, name the branch after
|
||||
it (e.g. `issue-11525`). This makes each PR's origin self-evident.
|
||||
- Otherwise, use a short, descriptive name with words separated by hyphens
|
||||
and no slashes (e.g. `fix-ellipse-icon-typo`, `feat-auto-link-libraries`).
|
||||
|
||||
Since PRs are squash-merged, the branch name does not survive into the
|
||||
commit history — what matters is the [PR title](#title-format).
|
||||
|
||||
### Format
|
||||
|
||||
#### Title
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# HIGHLIGHTS
|
||||
|
||||
## 2.17.0
|
||||
|
||||
- Background blur is here
|
||||
- WebGL rendering gets stronger
|
||||
- MCP connection status and more
|
||||
- Design tokens: more visible, more user-friendly
|
||||
|
||||
|
||||
## 2.16.0
|
||||
|
||||
- Design tokens in the design panel
|
||||
- Major community contributions
|
||||
- WebGL rendering (beta)
|
||||
|
||||
|
||||
## 2.15.0
|
||||
|
||||
- AI connected to real design context
|
||||
- Multi-directional workflow
|
||||
- Your stack, your model, your decision
|
||||
|
||||
|
||||
|
||||
|
||||
+17
-11
@@ -17,7 +17,7 @@
|
||||
|
||||
io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
|
||||
|
||||
io.lettuce/lettuce-core {:mvn/version "7.6.0.RELEASE"}
|
||||
io.lettuce/lettuce-core {:mvn/version "7.7.0.RELEASE"}
|
||||
;; Minimal dependencies required by lettuce, we need to include them
|
||||
;; explicitly because clojure dependency management does not support
|
||||
;; yet the BOM format.
|
||||
@@ -25,7 +25,7 @@
|
||||
io.micrometer/micrometer-observation {:mvn/version "1.14.2"}
|
||||
|
||||
java-http-clj/java-http-clj {:mvn/version "0.4.3"}
|
||||
com.google.guava/guava {:mvn/version "33.6.0-jre"}
|
||||
com.google.guava/guava {:mvn/version "33.7.1-jre"}
|
||||
|
||||
funcool/yetti
|
||||
{:git/tag "v11.10"
|
||||
@@ -40,38 +40,44 @@
|
||||
nrepl/nrepl {:mvn/version "1.7.0"}
|
||||
|
||||
org.postgresql/postgresql {:mvn/version "42.7.13"}
|
||||
org.xerial/sqlite-jdbc {:mvn/version "3.53.2.1"}
|
||||
org.xerial/sqlite-jdbc {:mvn/version "3.53.4.0"}
|
||||
|
||||
com.zaxxer/HikariCP {:mvn/version "7.1.0"}
|
||||
|
||||
io.whitfin/siphash {:mvn/version "2.0.0"}
|
||||
io.whitfin/siphash {:mvn/version "3.0.0"}
|
||||
|
||||
buddy/buddy-hashers {:mvn/version "2.0.167"}
|
||||
buddy/buddy-sign {:mvn/version "3.6.1-359"}
|
||||
org.passay/passay {:mvn/version "1.6.6"}
|
||||
org.passay/passay {:mvn/version "2.0.0"}
|
||||
|
||||
com.github.ben-manes.caffeine/caffeine {:mvn/version "3.2.4"}
|
||||
|
||||
org.jsoup/jsoup {:mvn/version "1.23.1"}
|
||||
org.jsoup/jsoup {:mvn/version "1.23.2"}
|
||||
|
||||
at.yawk.lz4/lz4-java
|
||||
{:mvn/version "1.11.1"}
|
||||
{:mvn/version "1.11.2"}
|
||||
|
||||
org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
|
||||
|
||||
dawran6/emoji {:mvn/version "0.2.0"}
|
||||
markdown-clj/markdown-clj {:mvn/version "1.12.8"}
|
||||
markdown-clj/markdown-clj {:mvn/version "1.12.9"}
|
||||
|
||||
;; Pretty Print specs
|
||||
pretty-spec/pretty-spec {:mvn/version "0.1.4"}
|
||||
software.amazon.awssdk/s3 {:mvn/version "2.50.1"}
|
||||
software.amazon.awssdk/sts {:mvn/version "2.50.1"}}
|
||||
software.amazon.awssdk/s3 {:mvn/version "2.54.5"}
|
||||
software.amazon.awssdk/sts {:mvn/version "2.54.5"}
|
||||
|
||||
com.ladybugdb/lbug {:mvn/version "0.19.1"}
|
||||
;; Required by Arrow RootAllocator (lbug only pulls arrow-memory-core).
|
||||
org.apache.arrow/arrow-memory-netty {:mvn/version "18.2.0"}}
|
||||
|
||||
:paths ["src" "resources" "target/classes"]
|
||||
:aliases
|
||||
{:dev
|
||||
{:jvm-opts ["--sun-misc-unsafe-memory-access=allow"
|
||||
"--enable-native-access=ALL-UNNAMED"]
|
||||
"--enable-native-access=ALL-UNNAMED"
|
||||
;; Arrow jars are on the classpath (unnamed module), not module-path.
|
||||
"--add-opens=java.base/java.nio=ALL-UNNAMED"]
|
||||
:extra-deps
|
||||
{com.bhauman/rebel-readline {:mvn/version "0.1.11"}
|
||||
clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
|
||||
|
||||
@@ -191,7 +191,7 @@
|
||||
file named “{{file-name|abbreviate:25}}”.
|
||||
</p>
|
||||
<p>
|
||||
Since this file is in your Penpot team, you can provide access by sending a view-only link.
|
||||
Since this file is in your Personal Projects, you can provide access by sending a view-only link.
|
||||
This will allow {{requested-by|abbreviate:25}} to view the content without making any changes.
|
||||
</p>
|
||||
<p>To proceed, please click the button below to generate and send the view-only link:</p>
|
||||
|
||||
@@ -2,7 +2,7 @@ Hello!
|
||||
|
||||
{{requested-by|abbreviate:25}} ({{requested-by-email}}) wants to have view-only access to the file named “{{file-name|abbreviate:25}}”.
|
||||
|
||||
Since this file is in your Penpot team, you can provide access by sending a view-only link. This will allow {{requested-by|abbreviate:25}} to view the content without making any changes.
|
||||
Since this file is in your Personal Projects, you can provide access by sending a view-only link. This will allow {{requested-by|abbreviate:25}} to view the content without making any changes.
|
||||
|
||||
To proceed, please click the link below to generate and send the view-only link:
|
||||
|
||||
|
||||
@@ -191,7 +191,7 @@
|
||||
“{{file-name|abbreviate:25}}”.
|
||||
</p>
|
||||
<p>
|
||||
Please note that the file is currently in Your Penpot 's team, so direct access cannot be
|
||||
Please note that the file is currently in Personal Projects, so direct access cannot be
|
||||
granted. However, you have two options to provide the requested access:
|
||||
</p>
|
||||
<ul>
|
||||
|
||||
@@ -5,7 +5,7 @@ Hello!
|
||||
|
||||
{{requested-by|abbreviate:25}} ({{requested-by-email}}) has requested access to the file named “{{file-name|abbreviate:25}}”.
|
||||
|
||||
Please note that the file is currently in Your Penpot 's team, so direct access cannot be granted. However, you have two options to provide the requested access:
|
||||
Please note that the file is currently in Personal Projects, so direct access cannot be granted. However, you have two options to provide the requested access:
|
||||
|
||||
- Move the File to Another Team:
|
||||
|
||||
|
||||
@@ -236,7 +236,23 @@ Debug Main Page
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
{% if graph-enabled %}
|
||||
<fieldset>
|
||||
<legend>Export graph (Ladybug):</legend>
|
||||
<desc>Given a FILE-ID, builds the graph projection and downloads
|
||||
the `.lbug` database file.</desc>
|
||||
|
||||
<form method="get" action="/dbg/actions/graph-export">
|
||||
<div class="row">
|
||||
<input type="text" style="width:300px" name="file-id" placeholder="file-id" />
|
||||
</div>
|
||||
<div class="row">
|
||||
<input type="submit" value="Download .lbug" />
|
||||
<a href="/dbg/graph">Open graph console</a>
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
{% endif %}
|
||||
<fieldset>
|
||||
<legend>Import binfile:</legend>
|
||||
<desc>Import penpot file in binary format.</desc>
|
||||
@@ -280,5 +296,60 @@ Debug Main Page
|
||||
</form>
|
||||
</fieldset>
|
||||
</section>
|
||||
|
||||
</main>
|
||||
|
||||
<main class="dashboard wide">
|
||||
<section class="widget wide">
|
||||
<fieldset>
|
||||
<legend>Export jobs:</legend>
|
||||
<desc>
|
||||
Export jobs as the exporter left them in redis. Records expire an hour
|
||||
after the export settles, so this is a live view, not a history.
|
||||
</desc>
|
||||
|
||||
<form method="get" action="/dbg">
|
||||
<div class="row">
|
||||
<input type="text" style="width:300px" name="job-id"
|
||||
placeholder="filter by job id" value="{{export-job-filter}}" />
|
||||
<input type="submit" value="Filter" />
|
||||
<a href="/dbg">clear</a>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div class="scroll-box">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>JOB ID</th>
|
||||
<th>STATE</th>
|
||||
<th>PROGRESS</th>
|
||||
<th>CMD</th>
|
||||
<th>BACKEND</th>
|
||||
<th>NAME</th>
|
||||
<th>CREATED</th>
|
||||
<th>ENDED</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for job in export-jobs %}
|
||||
<tr>
|
||||
<td><tt>{{job.id}}</tt></td>
|
||||
<td>{{job.state}}{% if job.interrupted %} (interrupted){% endif %}</td>
|
||||
<td>{{job.done}} / {{job.total}}</td>
|
||||
<td>{{job.cmd}}</td>
|
||||
<td>{{job.backend}}</td>
|
||||
<td>{{job.name}}</td>
|
||||
<td>{{job.created-at}}</td>
|
||||
<td>{{job.ended-at}}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tr><td colspan="8">No export jobs.</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</fieldset>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -143,6 +143,35 @@ nav > div:not(:last-child) {
|
||||
height: fit-content;
|
||||
}
|
||||
|
||||
/* A widget that holds a table rather than a form: full width, and tall
|
||||
enough to be worth scrolling inside. */
|
||||
.dashboard.wide {
|
||||
margin-top: 0px;
|
||||
}
|
||||
|
||||
.widget.wide {
|
||||
max-width: none;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.widget.wide .scroll-box {
|
||||
max-height: 320px;
|
||||
overflow-y: auto;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.widget.wide table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.widget.wide th {
|
||||
text-align: left;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: white;
|
||||
}
|
||||
|
||||
.widget input[type=submit] {
|
||||
outline: none;
|
||||
border: 1px solid gray;
|
||||
|
||||
@@ -48,7 +48,6 @@ export PENPOT_FLAGS="\
|
||||
enable-user-feedback \
|
||||
disable-secure-session-cookies \
|
||||
enable-smtp \
|
||||
enable-account-lockout \
|
||||
enable-prepl-server \
|
||||
enable-urepl-server \
|
||||
enable-nrepl-server \
|
||||
@@ -94,7 +93,8 @@ export JAVA_OPTS="\
|
||||
-XX:-OmitStackTraceInFastThrow \
|
||||
--sun-misc-unsafe-memory-access=allow \
|
||||
--enable-preview \
|
||||
--enable-native-access=ALL-UNNAMED";
|
||||
--enable-native-access=ALL-UNNAMED \
|
||||
--add-opens=java.base/java.nio=ALL-UNNAMED";
|
||||
|
||||
function setup_minio() {
|
||||
if [ "${PENPOT_OBJECTS_STORAGE_BACKEND}" != "s3" ]; then
|
||||
|
||||
@@ -18,7 +18,7 @@ if [ -f ./environ ]; then
|
||||
source ./environ
|
||||
fi
|
||||
|
||||
export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
|
||||
export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
|
||||
|
||||
ENTRYPOINT=${1:-app.main};
|
||||
|
||||
|
||||
@@ -14,10 +14,21 @@
|
||||
:iterations 3
|
||||
:parallelism 2})
|
||||
|
||||
(def ^:private weak-options
|
||||
{:alg :pbkdf2+sha256
|
||||
:iterations 100})
|
||||
|
||||
(defn derive-password
|
||||
[password]
|
||||
(hashers/derive password default-options))
|
||||
|
||||
(defn derive-password-weak
|
||||
"Derives a password using a fast algorithm (pbkdf2+sha256, 100 iterations).
|
||||
Intended for demo users only — they are already gated behind the
|
||||
`demo-users` config flag which is disabled in production."
|
||||
[password]
|
||||
(hashers/derive password weak-options))
|
||||
|
||||
(defn verify-password
|
||||
[attempt password]
|
||||
(try
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.auth.login-lockout
|
||||
"Brute-force protection: per-account failed login counter backed by Redis.
|
||||
Uses an atomic Lua script for the increment operation to prevent
|
||||
concurrent login bypass. Stores count:window_start in the value;
|
||||
time is passed from Clojure (ct/now) via ARGV, keeping it injectable
|
||||
for tests. When the counter reaches the configured threshold within
|
||||
the time window, the account is locked out until the window expires."
|
||||
(:require
|
||||
[app.common.generic-pool :as gpool]
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.config :as cf]
|
||||
[app.redis :as rds]
|
||||
[app.redis.script :as-alias rscript]
|
||||
[clojure.string :as str])
|
||||
(:import
|
||||
java.lang.AutoCloseable))
|
||||
|
||||
(def ^:private key-prefix "penpot.login-lockout.")
|
||||
|
||||
(def ^:private lockout-script
|
||||
{::rscript/name ::login-lockout
|
||||
::rscript/path "app/auth/login_lockout.lua"})
|
||||
|
||||
(defn- get-pool
|
||||
[cfg]
|
||||
(:app.redis/pool cfg))
|
||||
|
||||
(defn- with-conn
|
||||
[cfg f]
|
||||
(let [pool (get-pool cfg)
|
||||
conn (gpool/get pool)]
|
||||
(try
|
||||
(f @conn)
|
||||
(finally
|
||||
(.close ^AutoCloseable conn)))))
|
||||
|
||||
(defn- parse-value
|
||||
"Parse stored \"count:window_start\" string. Returns nil if missing
|
||||
or expired. Throws NumberFormatException for malformed values
|
||||
(caught by caller → fail-open)."
|
||||
[value window-ms now-ms]
|
||||
(when value
|
||||
(let [sep (str/index-of value ":")]
|
||||
(when sep
|
||||
(let [count (Long/parseLong (subs value 0 sep))
|
||||
start (Long/parseLong (subs value (inc sep)))]
|
||||
(when (> (+ start window-ms) now-ms)
|
||||
{:count count :window-start start}))))))
|
||||
|
||||
(defn record-failed-attempt!
|
||||
"Increment the failed-login counter for a profile-id (atomic via Lua).
|
||||
Returns nil when the flag is disabled or on Redis error (fail-open).
|
||||
Otherwise returns a map with :count (int), :ttl (int, seconds),
|
||||
and :locked? (boolean)."
|
||||
[cfg profile-id]
|
||||
(when (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(let [threshold (cf/get :login-lockout-max-attempts)
|
||||
window (cf/get :login-lockout-window)
|
||||
window-ms (if (integer? window) window (.toMillis window))
|
||||
_ (assert (>= threshold 1) "login-lockout-max-attempts must be >= 1")
|
||||
_ (assert (>= window-ms 60000) "login-lockout-window must be >= 60000ms (1 minute)")
|
||||
key (str key-prefix profile-id)
|
||||
now-ms (inst-ms (ct/now))
|
||||
result (with-conn cfg
|
||||
(fn [conn]
|
||||
(rds/eval conn
|
||||
(assoc lockout-script
|
||||
::rscript/keys [key]
|
||||
::rscript/vals [threshold window-ms now-ms]))))
|
||||
[count locked ttl] result]
|
||||
{:count count
|
||||
:locked? (= 1 locked)
|
||||
:ttl (max 0 ttl)})
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failing open on login lockout"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)
|
||||
nil))))
|
||||
|
||||
(defn clear-attempts!
|
||||
"Clear the failed-login counter (on successful login or password reset).
|
||||
No-op when the flag is disabled."
|
||||
[cfg profile-id]
|
||||
(when (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(with-conn cfg
|
||||
(fn [conn]
|
||||
(rds/del conn (str key-prefix profile-id))))
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failed to clear login lockout"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)))))
|
||||
|
||||
(defn locked?
|
||||
"Check whether the account is currently locked out. Does not increment
|
||||
the counter. Returns {:locked? false} when the flag is disabled or on
|
||||
Redis error (fail-open). When locked, includes :ttl (seconds remaining)."
|
||||
[cfg profile-id]
|
||||
(if (contains? cf/flags :account-lockout)
|
||||
(try
|
||||
(let [threshold (cf/get :login-lockout-max-attempts)
|
||||
window (cf/get :login-lockout-window)
|
||||
window-ms (if (integer? window) window (.toMillis window))
|
||||
_ (assert (>= threshold 1) "login-lockout-max-attempts must be >= 1")
|
||||
_ (assert (>= window-ms 60000) "login-lockout-window must be >= 60000ms (1 minute)")
|
||||
key (str key-prefix profile-id)
|
||||
now-ms (inst-ms (ct/now))
|
||||
result (with-conn cfg
|
||||
(fn [conn]
|
||||
(if-let [current (parse-value (rds/get conn key) window-ms now-ms)]
|
||||
(let [elapsed (- now-ms (:window-start current))
|
||||
ttl-ms (- window-ms elapsed)
|
||||
locked? (>= (:count current) threshold)]
|
||||
(cond-> {:locked? locked?}
|
||||
locked?
|
||||
(assoc :ttl (int (Math/ceil (/ ttl-ms 1000.0))))))
|
||||
{:locked? false})))]
|
||||
result)
|
||||
(catch Exception cause
|
||||
(l/warn :hint "redis unavailable, failing open on lockout check"
|
||||
:profile-id (str profile-id)
|
||||
:cause cause)
|
||||
{:locked? false}))
|
||||
{:locked? false}))
|
||||
@@ -1,46 +0,0 @@
|
||||
-- This Source Code Form is subject to the terms of the Mozilla Public
|
||||
-- License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
--
|
||||
-- Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
local key = KEYS[1]
|
||||
local threshold = tonumber(ARGV[1])
|
||||
local window_ms = tonumber(ARGV[2])
|
||||
local now_ms = tonumber(ARGV[3])
|
||||
local window_sec = math.ceil(window_ms / 1000)
|
||||
|
||||
local val = redis.call('GET', key)
|
||||
local count
|
||||
local window_start
|
||||
|
||||
if val then
|
||||
local colon = string.find(val, ':')
|
||||
if colon then
|
||||
count = tonumber(string.sub(val, 1, colon - 1))
|
||||
window_start = tonumber(string.sub(val, colon + 1))
|
||||
else
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
else
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
|
||||
if (now_ms - window_start) > window_ms then
|
||||
count = 0
|
||||
window_start = now_ms
|
||||
end
|
||||
|
||||
count = count + 1
|
||||
redis.call('SET', key, count .. ':' .. window_start, 'EX', window_sec)
|
||||
|
||||
local locked = 0
|
||||
local ttl = 0
|
||||
if count >= threshold then
|
||||
locked = 1
|
||||
ttl = math.ceil((window_ms - (now_ms - window_start)) / 1000)
|
||||
end
|
||||
|
||||
return {count, locked, ttl}
|
||||
@@ -9,7 +9,9 @@
|
||||
(:require
|
||||
[app.common.exceptions :as ex])
|
||||
(:import
|
||||
[org.passay CharacterCharacteristicsRule CharacterRule EnglishCharacterData PasswordData]))
|
||||
[org.passay PasswordData]
|
||||
[org.passay.data EnglishCharacterData]
|
||||
[org.passay.rule CharacterCharacteristicsRule CharacterRule]))
|
||||
|
||||
(defonce ^:private passay-code->translation-key
|
||||
{"INSUFFICIENT_LOWERCASE" "errors.weak-password.insufficient-lowercase"
|
||||
@@ -18,12 +20,13 @@
|
||||
"INSUFFICIENT_SPECIAL" "errors.weak-password.insufficient-special"})
|
||||
|
||||
(defonce ^:private character-characteristics-rule
|
||||
(doto (CharacterCharacteristicsRule.)
|
||||
(.setRules [(CharacterRule. EnglishCharacterData/LowerCase 1)
|
||||
(CharacterCharacteristicsRule.
|
||||
4
|
||||
(into-array org.passay.rule.CharacterRule
|
||||
[(CharacterRule. EnglishCharacterData/LowerCase 1)
|
||||
(CharacterRule. EnglishCharacterData/UpperCase 1)
|
||||
(CharacterRule. EnglishCharacterData/Digit 1)
|
||||
(CharacterRule. EnglishCharacterData/Special 1)])
|
||||
(.setNumberOfCharacteristics 4)))
|
||||
(CharacterRule. EnglishCharacterData/Special 1)])))
|
||||
|
||||
(defn validate-password
|
||||
"Validates password strength.
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
[datoteka.io :as io])
|
||||
(:import
|
||||
java.io.File
|
||||
java.io.FilterInputStream
|
||||
java.io.InputStream
|
||||
java.io.OutputStreamWriter
|
||||
java.lang.AutoCloseable
|
||||
@@ -461,6 +462,32 @@
|
||||
[^ZipFile input ^ZipEntry entry]
|
||||
(.getInputStream input entry))
|
||||
|
||||
(defn- size-limiting-stream
|
||||
"Wraps an InputStream to enforce a maximum number of decompressed bytes.
|
||||
Raises :validation :max-file-size-reached when the limit is exceeded."
|
||||
^InputStream
|
||||
[^InputStream input ^long max-size]
|
||||
(let [counter (atom 0)
|
||||
on-read (fn [n]
|
||||
(when (pos? n)
|
||||
(when (> (swap! counter + (long n)) max-size)
|
||||
(ex/raise :type :validation
|
||||
:code :max-file-size-reached
|
||||
:hint (str "stream exceeded max size: " max-size))))
|
||||
n)]
|
||||
(proxy [FilterInputStream] [input]
|
||||
(read
|
||||
([]
|
||||
(let [b (.read input)]
|
||||
(when (pos? b) (on-read 1))
|
||||
b))
|
||||
([^bytes buf]
|
||||
(on-read (.read input buf 0 (alength buf))))
|
||||
([^bytes buf off]
|
||||
(on-read (.read input buf (int off) (- (alength buf) (int off)))))
|
||||
([^bytes buf off len]
|
||||
(on-read (.read input buf (int off) (int len))))))))
|
||||
|
||||
(defn- zip-entry-reader
|
||||
[^ZipFile input ^ZipEntry entry]
|
||||
(-> (zip-entry-stream input entry)
|
||||
@@ -469,10 +496,12 @@
|
||||
(defn- zip-entry-storage-content
|
||||
"Wraps a ZipFile and ZipEntry into a penpot storage compatible
|
||||
object and avoid creating temporal objects"
|
||||
[input entry]
|
||||
(let [hash (delay (->> entry
|
||||
(zip-entry-stream input)
|
||||
(sto.impl/calculate-hash)))]
|
||||
[input entry & {:keys [max-size]}]
|
||||
(let [stream-fn (fn []
|
||||
(cond-> (zip-entry-stream input entry)
|
||||
max-size (size-limiting-stream max-size)))
|
||||
hash (delay (->> (stream-fn)
|
||||
(sto.impl/calculate-hash)))]
|
||||
(reify
|
||||
sto.impl/IContentObject
|
||||
(get-size [_]
|
||||
@@ -489,7 +518,7 @@
|
||||
(throw (UnsupportedOperationException. "not implemented")))
|
||||
|
||||
(make-input-stream [_ _]
|
||||
(zip-entry-stream input entry))
|
||||
(stream-fn))
|
||||
(make-output-stream [_ _]
|
||||
(throw (UnsupportedOperationException. "not implemented"))))))
|
||||
|
||||
@@ -884,9 +913,9 @@
|
||||
|
||||
ext (cmedia/mtype->extension (:content-type object))
|
||||
path (str "objects/" id ext)
|
||||
content (->> path
|
||||
(get-zip-entry input)
|
||||
(zip-entry-storage-content input))]
|
||||
content (zip-entry-storage-content input
|
||||
(get-zip-entry input path)
|
||||
:max-size (::bfc/import-max-object-size cfg))]
|
||||
|
||||
(when (not= (:size object) (sto/get-size content))
|
||||
(ex/raise :type :validation
|
||||
@@ -896,6 +925,15 @@
|
||||
:expected-size (:size object)
|
||||
:found-size (sto/get-size content)))
|
||||
|
||||
(when-let [max (::bfc/import-max-object-size cfg)]
|
||||
(when (> (sto/get-size content) max)
|
||||
(ex/raise :type :validation
|
||||
:code :max-file-size-reached
|
||||
:hint (str "storage object exceeds maximum size: " (sto/get-size content))
|
||||
:path path
|
||||
:max max
|
||||
:found (sto/get-size content))))
|
||||
|
||||
(when-let [hash (get object :hash)]
|
||||
(when (not= hash (sto/get-hash content))
|
||||
(ex/raise :type :validation
|
||||
@@ -1117,6 +1155,15 @@
|
||||
(let [manifest (-> (read-manifest input)
|
||||
(validate-manifest))
|
||||
entries (read-zip-entries input)
|
||||
|
||||
_ (when-let [max (::bfc/import-max-zip-entries cfg)]
|
||||
(when (> (count entries) max)
|
||||
(ex/raise :type :validation
|
||||
:code :too-many-zip-entries
|
||||
:hint (str "zip file has too many entries: " (count entries))
|
||||
:max max
|
||||
:found (count entries))))
|
||||
|
||||
cfg (-> cfg
|
||||
(assoc ::entries entries)
|
||||
(assoc ::manifest manifest)
|
||||
|
||||
@@ -69,9 +69,6 @@
|
||||
:profile-bounce-max-age (ct/duration {:days 7})
|
||||
:profile-bounce-threshold 10
|
||||
|
||||
:login-lockout-max-attempts 5
|
||||
:login-lockout-window (ct/duration "15m")
|
||||
|
||||
:telemetry-uri "https://telemetry.penpot.app/"
|
||||
|
||||
:media-max-file-size (* 1024 1024 30) ; 30MiB
|
||||
@@ -97,7 +94,11 @@
|
||||
|
||||
;; SSRF protection
|
||||
:ssrf-allowed-hosts #{}
|
||||
:ssrf-extra-blocked-cidrs #{}})
|
||||
:ssrf-extra-blocked-cidrs #{}
|
||||
|
||||
;; Binfile import limits
|
||||
:binfile-import-max-object-size (* 1024 1024 100) ;; 100 MiB
|
||||
:binfile-import-max-zip-entries (* 500 1000)}) ;; 500,000
|
||||
|
||||
(def schema:config
|
||||
(do #_sm/optional-keys
|
||||
@@ -154,8 +155,9 @@
|
||||
[:media-processing-service-uri {:optional true} ::sm/uri]
|
||||
[:media-processing-service-timeout {:optional true} ::sm/int]
|
||||
|
||||
[:login-lockout-max-attempts {:optional true} ::sm/int]
|
||||
[:login-lockout-window {:optional true} ::ct/duration]
|
||||
;; Binfile import limits (PENPOT_BINFILE_IMPORT_*)
|
||||
[:binfile-import-max-object-size {:optional true} ::sm/int]
|
||||
[:binfile-import-max-zip-entries {:optional true} ::sm/int]
|
||||
|
||||
[:deletion-delay {:optional true} ::ct/duration]
|
||||
[:file-clean-delay {:optional true} ::ct/duration]
|
||||
|
||||
@@ -31,8 +31,8 @@
|
||||
com.zaxxer.hikari.HikariDataSource
|
||||
com.zaxxer.hikari.HikariPoolMXBean
|
||||
com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
|
||||
io.whitfin.siphash.SipHasher
|
||||
io.whitfin.siphash.SipHasherContainer
|
||||
io.whitfin.siphash.SipHash
|
||||
io.whitfin.siphash.SipHashContext
|
||||
java.io.InputStream
|
||||
java.io.OutputStream
|
||||
java.sql.Connection
|
||||
@@ -701,12 +701,12 @@
|
||||
;; --- Locks
|
||||
|
||||
(def ^:private siphash-state
|
||||
(SipHasher/container
|
||||
(uuid/get-bytes uuid/zero)))
|
||||
(SipHash/context
|
||||
(uuid/get-bytes uuid/zero)))
|
||||
|
||||
(defn uuid->hash-code
|
||||
[o]
|
||||
(.hash ^SipHasherContainer siphash-state
|
||||
(.hash ^SipHashContext siphash-state
|
||||
^bytes (uuid/get-bytes o)))
|
||||
|
||||
(defn- xact-check-param
|
||||
|
||||
@@ -505,13 +505,13 @@
|
||||
:schema schema:request-file-access))
|
||||
|
||||
(def request-file-access-yourpenpot
|
||||
"File access on Your Penpot request email."
|
||||
"File access on Personal Projects request email."
|
||||
(template-factory
|
||||
:id ::request-file-access-yourpenpot
|
||||
:schema schema:request-file-access))
|
||||
|
||||
(def request-file-access-yourpenpot-view
|
||||
"File access on Your Penpot view mode request email."
|
||||
"File access on Personal Projects view mode request email."
|
||||
(template-factory
|
||||
:id ::request-file-access-yourpenpot-view
|
||||
:schema schema:request-file-access))
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.arrow
|
||||
"Bulk Ladybug ingest through in-memory Arrow.
|
||||
|
||||
Rows are built as Arrow `VectorSchemaRoot`s in the JVM's off-heap memory,
|
||||
handed to Ladybug as a virtual table, and `COPY`d into the real one. No file
|
||||
is written and no value is rendered as text for the engine to re-parse, so
|
||||
nothing in this path needs escaping. Arrow carries MAP, STRUCT, fixed-size
|
||||
arrays and multi-line strings natively.
|
||||
|
||||
The type language is Ladybug's, read recursively by `app.graph.schema.values`;
|
||||
this namespace adds the matching Arrow `Field` and a writer for each shape.
|
||||
`values/coerce` shapes a value first — a matrix into six doubles, a colour
|
||||
into a packed integer — exactly as it does for the Cypher path, so the two
|
||||
writers cannot disagree.
|
||||
|
||||
Engine facts this file depends on, each verified against lbug 0.19.1:
|
||||
|
||||
- An Arrow table is **not** a `COPY` source identifier, but it *is* a
|
||||
MATCH-able node label: `COPY T FROM (MATCH (n:stg) RETURN n.a AS a, …)`.
|
||||
- A MAP vector's `entries` child struct must be non-nullable, and
|
||||
`MapVector/getWriter` silently promotes it to a sparse union — so map
|
||||
vectors are built from an explicit `Field` and filled child-first.
|
||||
- Ladybug quotes the column and table names it interpolates into the staged
|
||||
table's DDL, and does not quote a STRUCT member name. So a top-level field
|
||||
arrives plain and a struct member whose name is a reserved word (`column`)
|
||||
arrives backticked.
|
||||
- `createArrowRelTable` resolves a UUID-keyed endpoint only from a
|
||||
`FixedSizeBinary(16)` column carrying the `arrow.uuid` extension, so edges
|
||||
are staged as a node table and joined by the `COPY` subquery instead."
|
||||
(:require
|
||||
[app.common.json :as json]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes]
|
||||
[app.graph.schema.values :as values]
|
||||
[clojure.string :as str])
|
||||
(:import
|
||||
com.ladybugdb.Connection
|
||||
com.ladybugdb.QueryResult
|
||||
java.nio.charset.StandardCharsets
|
||||
java.util.ArrayList
|
||||
java.util.List
|
||||
org.apache.arrow.memory.BufferAllocator
|
||||
org.apache.arrow.memory.RootAllocator
|
||||
org.apache.arrow.vector.BigIntVector
|
||||
org.apache.arrow.vector.BitVector
|
||||
org.apache.arrow.vector.complex.ListVector
|
||||
org.apache.arrow.vector.complex.MapVector
|
||||
org.apache.arrow.vector.complex.StructVector
|
||||
org.apache.arrow.vector.FieldVector
|
||||
org.apache.arrow.vector.Float8Vector
|
||||
org.apache.arrow.vector.TimeStampMicroVector
|
||||
org.apache.arrow.vector.types.FloatingPointPrecision
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Bool
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$FloatingPoint
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Int
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$List
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Map
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Struct
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Timestamp
|
||||
org.apache.arrow.vector.types.pojo.ArrowType$Utf8
|
||||
org.apache.arrow.vector.types.pojo.Field
|
||||
org.apache.arrow.vector.types.pojo.FieldType
|
||||
org.apache.arrow.vector.types.pojo.Schema
|
||||
org.apache.arrow.vector.types.TimeUnit
|
||||
org.apache.arrow.vector.UInt4Vector
|
||||
org.apache.arrow.vector.VarCharVector
|
||||
org.apache.arrow.vector.VectorSchemaRoot))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
;; --------------------------------------------------------------- allocator
|
||||
|
||||
(defn with-allocator!
|
||||
"Invoke `(f allocator)` with a fresh Arrow `RootAllocator`.
|
||||
|
||||
The allocator must outlive the Ladybug connection, because Ladybug releases
|
||||
its references to the staged buffers only when the Arrow tables are dropped —
|
||||
which happens on connection close at the latest. Closing it first surfaces as
|
||||
`IllegalStateException: Memory was leaked`, *thrown while unwinding*, which
|
||||
hides whatever actually failed. Any diagnostic here must catch inside this
|
||||
scope."
|
||||
[f]
|
||||
(with-open [allocator (RootAllocator.)]
|
||||
(f allocator)))
|
||||
|
||||
;; ------------------------------------------------------ Ladybug type → Field
|
||||
|
||||
(def ^:private scalar-arrow-type
|
||||
"Ladybug scalar → Arrow type. `UUID` and `JSON` ride as UTF-8: Ladybug
|
||||
accepts a string into either column and does the conversion itself, which is
|
||||
cheaper than teaching this side two more binary layouts."
|
||||
{"STRING" #(ArrowType$Utf8.)
|
||||
"UUID" #(ArrowType$Utf8.)
|
||||
"JSON" #(ArrowType$Utf8.)
|
||||
"INT64" #(ArrowType$Int. 64 true)
|
||||
"UINT32" #(ArrowType$Int. 32 false)
|
||||
"DOUBLE" #(ArrowType$FloatingPoint. FloatingPointPrecision/DOUBLE)
|
||||
"BOOLEAN" #(ArrowType$Bool.)
|
||||
"TIMESTAMP" #(ArrowType$Timestamp. TimeUnit/MICROSECOND nil)})
|
||||
|
||||
(defn column-field
|
||||
"Arrow `Field` for a column of `ladybug-type`, recursively.
|
||||
|
||||
`nullable?` is false only where Arrow's own invariants demand it — a MAP's
|
||||
`entries` struct and its key."
|
||||
(^Field [^String field-name ladybug-type]
|
||||
(column-field field-name ladybug-type true))
|
||||
(^Field [^String field-name ladybug-type nullable?]
|
||||
(cond
|
||||
;; A list first: `STRUCT(…)[]` starts with `STRUCT(` but is a list of them.
|
||||
(ladybug/list-type? ladybug-type)
|
||||
(Field. field-name (FieldType. nullable? (ArrowType$List.) nil)
|
||||
[(column-field "item" (values/list-element ladybug-type))])
|
||||
|
||||
(ladybug/map-type? ladybug-type)
|
||||
(let [[key-type value-type] (values/map-types ladybug-type)]
|
||||
(Field. field-name (FieldType. nullable? (ArrowType$Map. false) nil)
|
||||
[(Field. "entries" (FieldType. false (ArrowType$Struct.) nil)
|
||||
[(column-field "key" key-type false)
|
||||
(column-field "value" value-type)])]))
|
||||
|
||||
(ladybug/struct-type? ladybug-type)
|
||||
(Field. field-name (FieldType. nullable? (ArrowType$Struct.) nil)
|
||||
;; Backticks kept: Ladybug quotes none of these when it names the
|
||||
;; staged struct's fields, so `column` has to arrive quoted.
|
||||
(mapv (fn [[field field-type]] (column-field field field-type))
|
||||
(values/struct-fields-quoted ladybug-type)))
|
||||
|
||||
:else
|
||||
(if-let [mk (get scalar-arrow-type ladybug-type)]
|
||||
(Field. field-name (FieldType. nullable? (mk) nil) nil)
|
||||
(throw (ex-info (str "no Arrow mapping for Ladybug type: " ladybug-type)
|
||||
{:ladybug-type ladybug-type}))))))
|
||||
|
||||
;; ------------------------------------------------------------------- writer
|
||||
|
||||
(defn- utf8
|
||||
^bytes [v]
|
||||
(.getBytes (if (keyword? v) (name v) (str v)) StandardCharsets/UTF_8))
|
||||
|
||||
(defn- epoch-micros
|
||||
^long [v]
|
||||
(let [^java.time.Instant inst
|
||||
(cond
|
||||
(instance? java.time.Instant v) v
|
||||
(instance? java.util.Date v) (.toInstant ^java.util.Date v)
|
||||
:else (java.time.Instant/parse (str v)))]
|
||||
(+ (* (.getEpochSecond inst) 1000000) (long (quot (.getNano inst) 1000)))))
|
||||
|
||||
(defn- write-scalar!
|
||||
[^FieldVector fv ladybug-type ^long idx v]
|
||||
(case ladybug-type
|
||||
("STRING" "UUID") (.setSafe ^VarCharVector fv idx (utf8 v))
|
||||
;; A JSON column holds JSON, not a Clojure value's print form: `str` on a
|
||||
;; map yields `{:fill-color "#000000"}`, which is EDN and which every
|
||||
;; consumer of `fills`, `content` or `position_data` would fail to parse.
|
||||
;; Same encoder the Cypher path uses (`app.graph.ladybug/format-json`).
|
||||
"JSON" (.setSafe ^VarCharVector fv idx
|
||||
(.getBytes ^String (json/encode v)
|
||||
StandardCharsets/UTF_8))
|
||||
"INT64" (.setSafe ^BigIntVector fv idx (long v))
|
||||
"UINT32" (.setSafe ^UInt4Vector fv idx (unchecked-int (long v)))
|
||||
"DOUBLE" (.setSafe ^Float8Vector fv idx (double v))
|
||||
"BOOLEAN" (.setSafe ^BitVector fv idx (if v 1 0))
|
||||
"TIMESTAMP" (.setSafe ^TimeStampMicroVector fv idx (epoch-micros v))
|
||||
(throw (ex-info (str "no Arrow writer for Ladybug type: " ladybug-type)
|
||||
{:ladybug-type ladybug-type}))))
|
||||
|
||||
(defn write-value!
|
||||
"Write already-coerced `v` into `fv` at `idx`, per `ladybug-type`.
|
||||
|
||||
`map-key-fn` renders the keys of a `MAP(STRING, …)`, for the same reason
|
||||
`app.graph.ladybug/format-typed-value` takes one: the right spelling is a
|
||||
property of the column, not of the writer."
|
||||
;; `idx` is deliberately unhinted: Clojure only accepts primitive args on fns
|
||||
;; of four or fewer, and the map-key renderer has to travel with the value.
|
||||
[^FieldVector fv ladybug-type idx v map-key-fn]
|
||||
(if (nil? v)
|
||||
(.setNull fv (int idx))
|
||||
(cond
|
||||
(ladybug/list-type? ladybug-type)
|
||||
(let [^ListVector lv fv
|
||||
child (.getDataVector lv)
|
||||
element-type (values/list-element ladybug-type)
|
||||
elements (vec (if (or (sequential? v) (set? v)) v [v]))
|
||||
start (.startNewValue lv (int idx))]
|
||||
(dotimes [i (count elements)]
|
||||
(write-value! child element-type (+ start i) (nth elements i) map-key-fn))
|
||||
(.endValue lv (int idx) (count elements)))
|
||||
|
||||
(ladybug/map-type? ladybug-type)
|
||||
(let [^MapVector mv fv
|
||||
^StructVector entries (.getDataVector mv)
|
||||
[key-type value-type] (values/map-types ladybug-type)
|
||||
key-vec (.getChild entries "key")
|
||||
value-vec (.getChild entries "value")
|
||||
render-key (if (and map-key-fn (= "STRING" key-type)) map-key-fn identity)
|
||||
pairs (vec (seq v))
|
||||
start (.startNewValue mv (int idx))]
|
||||
(dotimes [i (count pairs)]
|
||||
(let [[k mv'] (nth pairs i)
|
||||
at (+ start i)]
|
||||
;; The entries struct is non-nullable: every slot must be defined.
|
||||
(.setIndexDefined entries (int at))
|
||||
(write-value! key-vec key-type at (render-key k) nil)
|
||||
(write-value! value-vec value-type at mv' map-key-fn)))
|
||||
(.endValue mv (int idx) (count pairs)))
|
||||
|
||||
(ladybug/struct-type? ladybug-type)
|
||||
(let [^StructVector sv fv]
|
||||
(.setIndexDefined sv (int idx))
|
||||
(doseq [[quoted-field field-type] (values/struct-fields-quoted ladybug-type)]
|
||||
;; The child is named with its backticks; the coerced value is keyed
|
||||
;; without them.
|
||||
(write-value! (.getChild sv quoted-field) field-type idx
|
||||
(get v (str/replace quoted-field "`" "")) map-key-fn)))
|
||||
|
||||
:else
|
||||
(write-scalar! fv ladybug-type (long idx) v))))
|
||||
|
||||
;; ------------------------------------------------------------------ batches
|
||||
|
||||
(defn- fill-vector!
|
||||
[^VectorSchemaRoot root ^String field-name ladybug-type rows value-fn map-key-fn]
|
||||
(let [^FieldVector fv (.getVector root field-name)]
|
||||
(.allocateNew fv)
|
||||
(dotimes [i (count rows)]
|
||||
(write-value! fv ladybug-type i
|
||||
(values/coerce ladybug-type (value-fn (nth rows i)))
|
||||
map-key-fn))
|
||||
(.setValueCount fv (count rows))))
|
||||
|
||||
(defn- node-batch
|
||||
"One `VectorSchemaRoot` holding every projected row of `table`.
|
||||
|
||||
Fields carry the plain column name. Ladybug quotes every identifier it
|
||||
interpolates into the staged table's DDL, so a name that is a reserved word
|
||||
(`Page.index`, `Document.options`) arrives unquoted and a name arriving
|
||||
pre-quoted comes out doubly backticked and fails to parse. The `COPY`
|
||||
projection below is Cypher, not DDL, so it quotes the same names itself."
|
||||
^VectorSchemaRoot [^BufferAllocator allocator table rows]
|
||||
(let [columns (nodes/column-keys table)
|
||||
fields (mapv (fn [k] (column-field (nodes/column-name table k)
|
||||
(nodes/column-ladybug-type table k)))
|
||||
columns)
|
||||
root (VectorSchemaRoot/create (Schema. ^List fields) allocator)]
|
||||
(doseq [k columns]
|
||||
(fill-vector! root (nodes/column-name table k)
|
||||
(nodes/column-ladybug-type table k)
|
||||
rows #(get % k) (nodes/column-map-key-fn table k)))
|
||||
(.setRowCount root (count rows))
|
||||
root))
|
||||
|
||||
(def ^:private edge-fields
|
||||
"Edge staging columns. `id` is the staging table's own key — Ladybug wants a
|
||||
first column to key the virtual table on — and `from`/`to` land as STRING,
|
||||
hence the cast in the join."
|
||||
[(Field. "id" (FieldType. true (ArrowType$Utf8.) nil) nil)
|
||||
(Field. "from" (FieldType. true (ArrowType$Utf8.) nil) nil)
|
||||
(Field. "to" (FieldType. true (ArrowType$Utf8.) nil) nil)
|
||||
(Field. "position" (FieldType. true (ArrowType$Int. 64 true) nil) nil)])
|
||||
|
||||
(defn- edge-batch
|
||||
^VectorSchemaRoot [^BufferAllocator allocator edges]
|
||||
(let [root (VectorSchemaRoot/create (Schema. ^List edge-fields) allocator)
|
||||
^VarCharVector iv (.getVector root "id")
|
||||
^VarCharVector fv (.getVector root "from")
|
||||
^VarCharVector tv (.getVector root "to")
|
||||
^BigIntVector pv (.getVector root "position")
|
||||
n (count edges)]
|
||||
(doseq [^FieldVector v [iv fv tv pv]] (.allocateNew v))
|
||||
(dotimes [i n]
|
||||
(let [{:keys [from-id to-id position]} (nth edges i)]
|
||||
(.setSafe iv i (utf8 i))
|
||||
(.setSafe fv i (utf8 from-id))
|
||||
(.setSafe tv i (utf8 to-id))
|
||||
(if (nil? position) (.setNull pv i) (.setSafe pv i (long position)))))
|
||||
(doseq [^FieldVector v [iv fv tv pv]] (.setValueCount v n))
|
||||
(.setRowCount root n)
|
||||
root))
|
||||
|
||||
;; ------------------------------------------------------------------ staging
|
||||
|
||||
(defn- batches
|
||||
^List [^VectorSchemaRoot root]
|
||||
(doto (ArrayList.) (.add root)))
|
||||
|
||||
(defn- check!
|
||||
[^QueryResult result hint data]
|
||||
(when-not (.isSuccess result)
|
||||
(throw (ex-info (str hint ": " (.getErrorMessage result))
|
||||
(assoc data :err (.getErrorMessage result))))))
|
||||
|
||||
(defn- with-staged-table!
|
||||
"Create Arrow table `staging-name` from `root`, run `(f)`, always drop it."
|
||||
[^Connection conn ^BufferAllocator allocator ^String staging-name
|
||||
^VectorSchemaRoot root data f]
|
||||
(try
|
||||
(with-open [^QueryResult r (.createArrowTable conn staging-name (batches root) allocator)]
|
||||
(check! r "createArrowTable failed" data))
|
||||
(f)
|
||||
(finally
|
||||
;; Dropped even on failure: the staged buffers stay referenced by Ladybug
|
||||
;; until it is, and the allocator's leak check fires on close otherwise.
|
||||
(try (.close ^QueryResult (.dropArrowTable conn staging-name))
|
||||
(catch Throwable _ nil)))))
|
||||
|
||||
(defn- copy-node-table!
|
||||
[^Connection conn table ^String staging-name]
|
||||
(let [projection (str/join ", " (for [k (nodes/column-keys table)
|
||||
:let [c (nodes/cypher-property-key table k)]]
|
||||
(str "n." c " AS " c)))
|
||||
statement (str "COPY `" table "` FROM (MATCH (n:" staging-name ") "
|
||||
"RETURN " projection ");")]
|
||||
(with-open [^QueryResult r (.query conn statement)]
|
||||
(check! r (str "COPY node table failed: " table)
|
||||
{:table table :statement statement}))))
|
||||
|
||||
(defn- copy-edge-group!
|
||||
"Load one FROM/TO pair of `IsChildOf`.
|
||||
|
||||
`createArrowRelTable` is unusable here — it cannot resolve endpoints against a
|
||||
UUID-keyed node table — so the edge list is staged as a node table and the
|
||||
endpoints are resolved by the subquery. The `WHERE` is clause-level because
|
||||
this dialect prohibits an inline pattern `WHERE`, and both sides are pinned by
|
||||
label so the join cannot reach outside the pair."
|
||||
[^Connection conn from-table to-table ^String staging-name]
|
||||
(let [statement (str "COPY `IsChildOf` FROM ("
|
||||
"MATCH (e:" staging-name "), "
|
||||
"(a:" (nodes/match-label from-table) "), "
|
||||
"(b:" (nodes/match-label to-table) ") "
|
||||
"WHERE a.id = cast(e.from AS UUID) "
|
||||
"AND b.id = cast(e.to AS UUID) "
|
||||
"RETURN a.id, b.id, e.position) "
|
||||
"(from='" from-table "', to='" to-table "');")]
|
||||
(with-open [^QueryResult r (.query conn statement)]
|
||||
(check! r (str "COPY edge group failed: " from-table " -> " to-table)
|
||||
{:from-table from-table :to-table to-table :statement statement}))))
|
||||
|
||||
(defn- staging-name
|
||||
[prefix & parts]
|
||||
(str/replace (str/join "_" (cons (str "stg_" prefix) parts)) #"[^A-Za-z0-9_]" "_"))
|
||||
|
||||
;; --------------------------------------------------------------------- load
|
||||
|
||||
(defn load-projection!
|
||||
"Load projected nodes and edges into an open Ladybug connection.
|
||||
|
||||
`allocator` must outlive `conn` — see `with-allocator!`."
|
||||
[^Connection conn {:keys [nodes edges]} ^BufferAllocator allocator]
|
||||
(doseq [[table rows] (sort-by key nodes)
|
||||
:when (seq rows)]
|
||||
(let [name (staging-name "node" table)]
|
||||
(with-open [root (node-batch allocator table rows)]
|
||||
(with-staged-table! conn allocator name root {:table table}
|
||||
#(copy-node-table! conn table name)))))
|
||||
(doseq [[[from-table to-table] group]
|
||||
(sort-by key (group-by (juxt :from-table :to-table) edges))
|
||||
:when (seq group)]
|
||||
(let [name (staging-name "edge" from-table to-table)]
|
||||
(with-open [root (edge-batch allocator group)]
|
||||
(with-staged-table! conn allocator name root
|
||||
{:from-table from-table :to-table to-table}
|
||||
#(copy-edge-group! conn from-table to-table name))))))
|
||||
@@ -0,0 +1,383 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.debug
|
||||
"In-memory Ladybug sessions for the debug graph console."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.graph.ingest :as graph.ingest]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes]
|
||||
[app.graph.sync :as graph.sync]
|
||||
[app.msgbus :as mbus]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[promesa.exec.csp :as sp])
|
||||
(:import
|
||||
com.ladybugdb.Connection
|
||||
com.ladybugdb.Database))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def default-query
|
||||
"Default console query, written to be self-explanatory in the textarea.
|
||||
The `filter_*` columns carry node ids for the graph-view result filter;
|
||||
the results table hides them (see `hide-filter-columns` and the
|
||||
template's `renderQueryOutput`)."
|
||||
(str "MATCH (s)-[r]->(t)\n"
|
||||
"// WHERE some condition\n"
|
||||
"RETURN label(s) AS src, s.name,\n"
|
||||
" label(r) AS rel,\n"
|
||||
" t.name, label(t) AS tgt,\n"
|
||||
"\n"
|
||||
"// filter_* columns omitted from table; these needed for graph view\n"
|
||||
"s.id AS filter_src_id, t.id AS filter_tgt_id;"))
|
||||
|
||||
(defonce ^:private sessions
|
||||
(atom {}))
|
||||
|
||||
(defn- session-key
|
||||
[profile-id]
|
||||
(str profile-id))
|
||||
|
||||
(defn- destroy-session!
|
||||
[{:keys [conn db sync-ch msgbus]}]
|
||||
(when sync-ch
|
||||
(sp/close! sync-ch)
|
||||
(when msgbus
|
||||
(mbus/purge! msgbus [sync-ch])))
|
||||
(when conn
|
||||
(ex/ignoring (.close ^Connection conn)))
|
||||
(when db
|
||||
(ex/ignoring (.close ^Database db))))
|
||||
|
||||
(defn- slim-ingest-meta
|
||||
"Drop full projection rows from session meta.
|
||||
|
||||
`build-index` needs `:nodes`/`:edges` once; keeping them in the session
|
||||
duplicates the entire graph on the JVM heap for every Load."
|
||||
[meta]
|
||||
(update meta :projection #(select-keys % [:stats])))
|
||||
|
||||
(defn- format-cell
|
||||
[value]
|
||||
(cond
|
||||
(nil? value) "NULL"
|
||||
(string? value) value
|
||||
:else (str value)))
|
||||
|
||||
(defn- format-query-result
|
||||
[{:keys [columns rows truncated?]}]
|
||||
{:columns (mapv str columns)
|
||||
:rows (mapv (fn [row]
|
||||
(mapv format-cell row))
|
||||
rows)
|
||||
:truncated? truncated?
|
||||
:row-count (count rows)})
|
||||
|
||||
(defn- apply-file-change!
|
||||
[conn profile-id {:keys [changes revn file-id]}]
|
||||
(try
|
||||
(some-> (get @sessions (session-key profile-id))
|
||||
(as-> current
|
||||
(when (= file-id (:file-id current))
|
||||
(let [lock (:lock current)
|
||||
result (locking lock
|
||||
(graph.sync/apply-changes!
|
||||
conn (:index current) changes revn))
|
||||
sync-at (ct/now)]
|
||||
(swap! sessions assoc-in [(session-key profile-id) :index]
|
||||
(:index result))
|
||||
(swap! sessions update-in [(session-key profile-id) :meta]
|
||||
(fn [meta]
|
||||
(cond-> (-> meta
|
||||
(update :sync dissoc :error)
|
||||
(assoc-in [:sync :last-at] sync-at)
|
||||
(assoc-in [:sync :last-applied] (:applied result))
|
||||
(assoc-in [:sync :last-skipped] (:skipped result)))
|
||||
(seq (:applied result))
|
||||
(assoc :revn (:revn result)))))
|
||||
(when (seq (:skipped result))
|
||||
(l/dbg :hint "graph sync skipped changes"
|
||||
:file-id (str file-id)
|
||||
:revn revn
|
||||
:skipped (:skipped result)))))))
|
||||
(catch Throwable cause
|
||||
(l/wrn :hint "graph sync failed"
|
||||
:file-id (str file-id)
|
||||
:cause cause)
|
||||
(swap! sessions assoc-in [(session-key profile-id) :meta :sync :error]
|
||||
(ex-message cause)))))
|
||||
|
||||
(defn- start-sync-loop!
|
||||
[{:keys [conn profile-id file-id] :as session}]
|
||||
(if-let [msgbus (:msgbus session)]
|
||||
(let [sync-ch (sp/chan :buf (sp/dropping-buffer 64))]
|
||||
(mbus/sub! msgbus :topic file-id :chan sync-ch)
|
||||
;; Recur ONLY while the channel is open. A bare `(recur)` after
|
||||
;; `take!` returns nil would spin forever and pin this Connection
|
||||
;; (and its Ladybug Database native memory) across every Load.
|
||||
(sp/go-loop []
|
||||
(when-let [message (sp/take! sync-ch)]
|
||||
(when (= :file-change (:type message))
|
||||
(apply-file-change! conn profile-id message))
|
||||
(recur)))
|
||||
(assoc session :sync-ch sync-ch))
|
||||
session))
|
||||
|
||||
(defn session-info
|
||||
"Return a public view of the current session for `profile-id`, if any."
|
||||
[profile-id]
|
||||
(when-let [{:keys [file-id meta loaded-at index]} (get @sessions (session-key profile-id))]
|
||||
{:file-id file-id
|
||||
:name (:name meta)
|
||||
:revn (:revn meta)
|
||||
:graph-revn (:revn index)
|
||||
:schema-version (:schema-version meta)
|
||||
:projection (:projection meta)
|
||||
:sync (:sync meta)
|
||||
:loaded-at (ct/format-inst loaded-at :iso)}))
|
||||
|
||||
(defn sync-status
|
||||
"Return incremental sync status for the active session."
|
||||
[profile-id]
|
||||
(when-let [session (get @sessions (session-key profile-id))]
|
||||
(let [{:keys [file-id meta index loaded-at]} session]
|
||||
{:file-id file-id
|
||||
:revn (:revn meta)
|
||||
:graph-revn (:revn index)
|
||||
:sync (:sync meta)
|
||||
:loaded-at (ct/format-inst loaded-at :iso)})))
|
||||
|
||||
(defn unload-session!
|
||||
"Close and discard the in-memory graph for `profile-id`."
|
||||
[profile-id]
|
||||
(when-let [session (get @sessions (session-key profile-id))]
|
||||
(destroy-session! session))
|
||||
(swap! sessions dissoc (session-key profile-id)))
|
||||
|
||||
(defn load-session!
|
||||
"Ingest `file-id` into a new in-memory Ladybug database for `profile-id`."
|
||||
[cfg profile-id file-id]
|
||||
(unload-session! profile-id)
|
||||
(let [^Database db (Database.)
|
||||
^Connection conn (Connection. db)
|
||||
msgbus (::mbus/msgbus cfg)]
|
||||
(.setQueryTimeout conn 0)
|
||||
(ladybug/ensure-extensions! conn)
|
||||
(try
|
||||
(let [meta (graph.ingest/ingest-on-connection! cfg conn file-id
|
||||
:db-path ":memory:"
|
||||
:skip-stats? true
|
||||
:skip-validation? true)
|
||||
index (graph.sync/build-index file-id (:revn meta) (:projection meta))
|
||||
;; Discard projection rows after indexing — they are only needed
|
||||
;; to seed the sync index and would otherwise leak heap on each Load.
|
||||
meta (slim-ingest-meta meta)
|
||||
session
|
||||
;; :lock serializes access to the shared Connection between the
|
||||
;; msgbus sync loop (writes) and HTTP handlers (reads); the Java
|
||||
;; binding gives no thread-safety guarantee for one Connection.
|
||||
(-> {:db db
|
||||
:conn conn
|
||||
:lock (Object.)
|
||||
:file-id file-id
|
||||
:meta meta
|
||||
:index index
|
||||
:msgbus msgbus
|
||||
:profile-id profile-id
|
||||
:loaded-at (ct/now)}
|
||||
start-sync-loop!)]
|
||||
(swap! sessions assoc (session-key profile-id) session)
|
||||
meta)
|
||||
(catch Throwable cause
|
||||
(destroy-session! {:conn conn :db db :msgbus msgbus})
|
||||
(throw cause)))))
|
||||
|
||||
(defn query-session!
|
||||
"Run a read-only `statement` against the in-memory graph for `profile-id`.
|
||||
|
||||
The statement is bound against the live schema before it runs, so a query
|
||||
naming a table or a property that does not exist reports the binder's own
|
||||
message and executes nothing. The engine's read/write analysis then decides
|
||||
whether it may run at all: the console is an inspection surface, and a
|
||||
session graph is rebuilt from the file by Reload, so a mutation from here
|
||||
would produce a graph no rebuild reproduces."
|
||||
[profile-id statement]
|
||||
(when (str/blank? statement)
|
||||
(ex/raise :type :validation
|
||||
:code :missing-query
|
||||
:hint "cypher query is required"))
|
||||
(if-let [{:keys [conn lock]} (get @sessions (session-key profile-id))]
|
||||
(locking lock
|
||||
(let [{:keys [ok? error read-only?]} (ladybug/validate-on-connection! conn statement)]
|
||||
(when-not ok?
|
||||
(ex/raise :type :validation
|
||||
:code :graph-query-invalid
|
||||
:hint error))
|
||||
(when-not read-only?
|
||||
(ex/raise :type :validation
|
||||
:code :graph-query-not-read-only
|
||||
:hint "the graph console runs read-only queries"))
|
||||
(-> (ladybug/query-on-connection! conn statement)
|
||||
format-query-result)))
|
||||
(ex/raise :type :not-found
|
||||
:code :graph-session-not-loaded
|
||||
:hint "load a file graph before running queries")))
|
||||
|
||||
(def ^:private export-max-rows
|
||||
"Row cap for graph-view export queries; far above expected per-file node
|
||||
and edge counts. `:truncated` in the export signals when it was hit."
|
||||
100000)
|
||||
|
||||
(defn- export-nodes
|
||||
[conn]
|
||||
(reduce
|
||||
(fn [acc {:keys [table]}]
|
||||
(let [stmt (str "MATCH (n:" (nodes/match-label table)
|
||||
") RETURN n.id AS id, n.name AS name;")
|
||||
{:keys [rows truncated?]}
|
||||
(ladybug/query-on-connection! conn stmt :max-rows export-max-rows)]
|
||||
(-> acc
|
||||
(update :nodes into
|
||||
(map (fn [[id label]]
|
||||
{:id (str id) :label (str label) :table table}))
|
||||
rows)
|
||||
(update :truncated? #(or % truncated?)))))
|
||||
{:nodes [] :truncated? false}
|
||||
nodes/node-types))
|
||||
|
||||
(defn rel-tables
|
||||
"Every relationship table in the open database, with whether it carries a
|
||||
`position` property.
|
||||
|
||||
Read from the catalog rather than listed here, so a newly ported transform's
|
||||
rel table appears in the graph view without the console being told about it."
|
||||
[conn]
|
||||
(for [[table] (:rows (ladybug/query-on-connection!
|
||||
conn "CALL show_tables() WHERE type = 'REL' RETURN name;"
|
||||
:max-rows 1000))
|
||||
:let [props (->> (ladybug/query-on-connection!
|
||||
conn (str "CALL table_info('" table "') RETURN *;")
|
||||
:max-rows 1000)
|
||||
:rows
|
||||
(into #{} (map (comp str second))))]]
|
||||
{:table table :position? (contains? props "position")}))
|
||||
|
||||
(defn- export-edges
|
||||
[conn]
|
||||
(reduce
|
||||
(fn [acc {:keys [table position?]}]
|
||||
(let [stmt (str "MATCH (a)-[r:`" table "`]->(b) "
|
||||
"RETURN a.id AS source, b.id AS target, "
|
||||
(if position? "r.position" "NULL") " AS position, "
|
||||
"'" table "' AS rel;")
|
||||
{:keys [rows truncated?]}
|
||||
(ladybug/query-on-connection! conn stmt :max-rows export-max-rows)]
|
||||
(-> acc
|
||||
(update :edges into
|
||||
(map (fn [[source target position rel]]
|
||||
(cond-> {:source (str source)
|
||||
:target (str target)
|
||||
:rel (str rel)}
|
||||
(some? position) (assoc :position position))))
|
||||
rows)
|
||||
(update :truncated? #(or % truncated?)))))
|
||||
{:edges [] :truncated? false}
|
||||
(rel-tables conn)))
|
||||
|
||||
(defn- bm-usage-bytes
|
||||
"Buffer-manager memory in use by this session's in-memory database
|
||||
(`CALL bm_info()` → [mem_limit mem_usage]); nil if the call fails."
|
||||
[conn]
|
||||
(ex/ignoring
|
||||
(-> (ladybug/query-on-connection! conn "CALL bm_info() RETURN *;" :max-rows 1)
|
||||
:rows first second)))
|
||||
|
||||
(defn export-graph-data!
|
||||
"Export the node/edge inventory of the in-memory graph for `profile-id`
|
||||
as plain data for the debug graph view. Returns nil when no session is
|
||||
loaded. Queries the Ladybug database (not the sync index) so the view
|
||||
reflects actual DB state, including drift."
|
||||
[profile-id]
|
||||
(when-let [{:keys [conn lock file-id index]} (get @sessions (session-key profile-id))]
|
||||
(locking lock
|
||||
(let [{:keys [nodes] nodes-truncated? :truncated?} (export-nodes conn)
|
||||
{:keys [edges] edges-truncated? :truncated?} (export-edges conn)]
|
||||
{:file-id (str file-id)
|
||||
:revn (:revn index)
|
||||
:truncated (boolean (or nodes-truncated? edges-truncated?))
|
||||
:bm-bytes (bm-usage-bytes conn)
|
||||
:nodes nodes
|
||||
:edges edges}))))
|
||||
|
||||
(defn- delete-tree!
|
||||
[^java.io.File file]
|
||||
(when (.exists file)
|
||||
(doseq [f (reverse (file-seq file))]
|
||||
(.delete ^java.io.File f))))
|
||||
|
||||
(defn export-session-database!
|
||||
"Materialize the in-memory session graph of `profile-id` as a `.lbug` file.
|
||||
|
||||
The console's graph is in-memory and live-synced, so it can differ from a
|
||||
fresh projection of the same file — which is exactly when someone wants to
|
||||
take it away and query it elsewhere. There is no \"save this database\"
|
||||
primitive, so the transfer goes through Ladybug's `EXPORT DATABASE` (Parquet
|
||||
per table) into a fresh on-disk database via `IMPORT DATABASE`.
|
||||
|
||||
Note the round trip drops table comments. Nothing in the graph is addressed
|
||||
by a table comment: every table is resolved by name, so the loss costs
|
||||
nothing.
|
||||
|
||||
Returns the path of the written database, or nil when no session is loaded.
|
||||
The caller owns the file and must delete it once streamed."
|
||||
[profile-id]
|
||||
(when-let [{:keys [conn lock file-id]} (get @sessions (session-key profile-id))]
|
||||
(let [stamp (System/nanoTime)
|
||||
staging (io/file (System/getProperty "java.io.tmpdir")
|
||||
(str "penpot-graph-session-" file-id "-" stamp))
|
||||
db-path (str (io/file (System/getProperty "java.io.tmpdir")
|
||||
(str file-id "-session-" stamp ".lbug")))]
|
||||
(try
|
||||
(locking lock
|
||||
(ladybug/exec-on-connection!
|
||||
conn [(str "EXPORT DATABASE '" (.getAbsolutePath staging)
|
||||
"' (format='parquet');")]))
|
||||
(ladybug/with-connection! db-path
|
||||
(fn [target]
|
||||
(ladybug/exec-on-connection!
|
||||
target [(str "IMPORT DATABASE '" (.getAbsolutePath staging) "';")
|
||||
"CHECKPOINT;"])))
|
||||
db-path
|
||||
(finally
|
||||
(delete-tree! staging))))))
|
||||
|
||||
(defn- hide-filter-columns
|
||||
"Drop `filter_*` columns from a query result before HTML table render;
|
||||
they exist to feed node ids to the graph-view filter, not for reading.
|
||||
The JSON response path keeps the full result."
|
||||
[{:keys [columns rows] :as result}]
|
||||
(let [idxs (vec (keep-indexed
|
||||
(fn [i c] (when-not (str/starts-with? (str c) "filter_") i))
|
||||
columns))]
|
||||
(if (or (empty? idxs) (= (count idxs) (count columns)))
|
||||
result
|
||||
(assoc result
|
||||
:columns (mapv (vec columns) idxs)
|
||||
:rows (mapv (fn [row] (mapv (vec row) idxs)) rows)))))
|
||||
|
||||
(defn console-context
|
||||
"Build template data for the graph debug console page."
|
||||
[profile-id & {:keys [query query-result error message]}]
|
||||
{:session (session-info profile-id)
|
||||
:query (or query default-query)
|
||||
:query-result (some-> query-result hide-filter-columns)
|
||||
:error error
|
||||
:message message
|
||||
:default-query default-query})
|
||||
@@ -0,0 +1,106 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.ingest
|
||||
"Penpot file -> Ladybug graph projection."
|
||||
(:require
|
||||
[app.binfile.common :as bfc]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.db :as db]
|
||||
[app.graph.arrow :as graph.arrow]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.meta :as graph.meta]
|
||||
[app.graph.projection.document :as projection.document]
|
||||
[app.graph.projection.transforms :as projection.transforms]
|
||||
[app.graph.schema :as schema]
|
||||
[app.graph.stats :as stats]
|
||||
[app.srepl.helpers :as h])
|
||||
(:import
|
||||
com.ladybugdb.Connection
|
||||
org.apache.arrow.memory.BufferAllocator))
|
||||
|
||||
(defn- fetch-file!
|
||||
[system file-id]
|
||||
(let [file-id (h/parse-uuid file-id)
|
||||
file (db/run! system #(bfc/get-file % file-id :realize? true))]
|
||||
(when-not file
|
||||
(ex/raise :type :not-found
|
||||
:code :file-not-found
|
||||
:file-id (str file-id)))
|
||||
(when-not (:data file)
|
||||
(ex/raise :type :validation
|
||||
:code :file-without-data
|
||||
:hint "file has no data to project"
|
||||
:file-id (str file-id)))
|
||||
[file-id file]))
|
||||
|
||||
(defn- ingest-on-connection*!
|
||||
[system ^Connection conn file-id ^BufferAllocator allocator
|
||||
{:keys [db-path skip-stats? skip-validation?] :or {skip-stats? true}}]
|
||||
(let [[file-id file] (fetch-file! system file-id)
|
||||
db-path (or db-path (ladybug/db-path-for-file file-id))
|
||||
data (:data file)]
|
||||
(when-not skip-validation?
|
||||
(ctf/check-file-data data))
|
||||
(l/inf :hint "graph ingest"
|
||||
:file-id (str file-id)
|
||||
:revn (:revn file)
|
||||
:db-path db-path
|
||||
:schema schema/schema-version)
|
||||
(let [ddl (schema/ddl-statements)
|
||||
{:keys [nodes edges stats]}
|
||||
(projection.document/projection-data data file)]
|
||||
(ladybug/exec-on-connection! conn ddl)
|
||||
(graph.arrow/load-projection! conn {:nodes nodes :edges edges} allocator)
|
||||
(ladybug/exec-on-connection! conn ["CHECKPOINT;"])
|
||||
(let [transforms (projection.transforms/apply-transforms! system conn data file)]
|
||||
;; Written last: its presence doubles as the build-complete marker.
|
||||
(graph.meta/write! conn {:file-id file-id
|
||||
:revn (:revn file)})
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:name (or (:name data) (:name file))
|
||||
:db-path db-path
|
||||
:schema-version schema/schema-version
|
||||
:projection {:stats stats
|
||||
:nodes nodes
|
||||
:edges edges}
|
||||
:transforms transforms
|
||||
:stats (when-not skip-stats?
|
||||
(stats/summarize-connection conn))}))))
|
||||
|
||||
(defn ingest-on-connection!
|
||||
"Project `file-id` into an already open Ladybug `conn`.
|
||||
|
||||
Takes an `:arrow-alloc` when the caller already owns one; otherwise it makes
|
||||
a short-lived allocator around this call. A caller that opened the connection
|
||||
itself should pass its own, because the allocator has to be closed *after*
|
||||
the connection — see `app.graph.arrow/with-allocator!`."
|
||||
[system ^Connection conn file-id & {:keys [arrow-alloc] :as opts}]
|
||||
(if arrow-alloc
|
||||
(ingest-on-connection*! system conn file-id arrow-alloc opts)
|
||||
(graph.arrow/with-allocator!
|
||||
(fn [allocator] (ingest-on-connection*! system conn file-id allocator opts)))))
|
||||
|
||||
(defn ingest-file!
|
||||
[system file-id & {:keys [db-path reset-db? skip-stats? skip-validation?]
|
||||
:or {reset-db? true}}]
|
||||
(let [db-path (or db-path (ladybug/db-path-for-file (h/parse-uuid file-id)))]
|
||||
(when reset-db?
|
||||
(ladybug/reset-db-path! db-path))
|
||||
;; Allocator outermost: Ladybug holds the staged Arrow buffers until its
|
||||
;; tables are dropped, which is no later than connection close, so the
|
||||
;; allocator must be closed after the connection and the database.
|
||||
(graph.arrow/with-allocator!
|
||||
(fn [allocator]
|
||||
(ladybug/with-connection! db-path
|
||||
(fn [conn]
|
||||
(ingest-on-connection*! system conn file-id allocator
|
||||
{:db-path db-path
|
||||
:skip-stats? skip-stats?
|
||||
:skip-validation? skip-validation?})))))))
|
||||
@@ -0,0 +1,504 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.ladybug
|
||||
"Ladybug access layer for graph-backed Penpot.
|
||||
|
||||
Uses the embedded Java API (`com.ladybugdb/lbug`)."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.json :as json]
|
||||
[app.graph.schema.values :as values]
|
||||
[clojure.string :as str]
|
||||
[datoteka.fs :as fs])
|
||||
(:import
|
||||
com.ladybugdb.Connection
|
||||
com.ladybugdb.Database
|
||||
com.ladybugdb.FlatTuple
|
||||
com.ladybugdb.PreparedStatement
|
||||
com.ladybugdb.QueryResult
|
||||
com.ladybugdb.Value))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn default-graph-dir
|
||||
[]
|
||||
(or (System/getenv "PENPOT_GRAPH_DIR") "/tmp/penpot-graph"))
|
||||
|
||||
(defn db-path-for-file
|
||||
[file-id]
|
||||
(str (fs/path (default-graph-dir) (str file-id ".lbug"))))
|
||||
|
||||
(defn- memory-db-path?
|
||||
[db-path]
|
||||
(= db-path ":memory:"))
|
||||
|
||||
(defn reset-db-path!
|
||||
[db-path]
|
||||
(when-not (memory-db-path? db-path)
|
||||
(when (fs/exists? db-path)
|
||||
(fs/delete db-path))))
|
||||
|
||||
(defn escape-cypher-string
|
||||
[s]
|
||||
(-> (str s)
|
||||
(str/replace "\\" "\\\\")
|
||||
(str/replace "'" "\\'")))
|
||||
|
||||
(defn format-uuid
|
||||
[id]
|
||||
(str "uuid('" (str id) "')"))
|
||||
|
||||
(defn format-string
|
||||
[s]
|
||||
(str "'" (escape-cypher-string s) "'"))
|
||||
|
||||
(defn format-int
|
||||
[n]
|
||||
(str (long n)))
|
||||
|
||||
(defn format-number
|
||||
[n]
|
||||
(if (== n (long n))
|
||||
(format-int n)
|
||||
(str (double n))))
|
||||
|
||||
(defn format-json
|
||||
[v]
|
||||
(str "json('" (escape-cypher-string (json/encode v)) "')"))
|
||||
|
||||
(defn format-timestamp
|
||||
"Ladybug TIMESTAMP literal of the form `timestamp('<ISO-8601 instant>')`."
|
||||
[v]
|
||||
(let [s (cond
|
||||
(instance? java.time.Instant v)
|
||||
(.toString ^java.time.Instant v)
|
||||
|
||||
(instance? java.util.Date v)
|
||||
(.toString (.toInstant ^java.util.Date v))
|
||||
|
||||
(string? v)
|
||||
v
|
||||
|
||||
:else
|
||||
(str v))]
|
||||
(str "timestamp('" (escape-cypher-string s) "')")))
|
||||
|
||||
(defn format-value
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) "NULL"
|
||||
(uuid? v) (format-uuid v)
|
||||
(instance? java.time.Instant v) (format-timestamp v)
|
||||
(instance? java.util.Date v) (format-timestamp v)
|
||||
(string? v) (format-string v)
|
||||
(number? v) (format-number v)
|
||||
(boolean? v) (if v "true" "false")
|
||||
(keyword? v) (format-string (name v))
|
||||
(map? v) (format-json v)
|
||||
(coll? v) (format-json v)
|
||||
:else (format-string (str v))))
|
||||
|
||||
(defn map-type?
|
||||
"Is `ladybug-type` a MAP column?"
|
||||
[ladybug-type]
|
||||
(and (string? ladybug-type)
|
||||
(str/starts-with? ladybug-type "MAP(")
|
||||
(not (str/ends-with? ladybug-type "]"))))
|
||||
|
||||
(defn list-type?
|
||||
"Is this a list or fixed-size array type? Checked before MAP and STRUCT,
|
||||
since `STRUCT(…)[]` starts with `STRUCT(` but is a list of them."
|
||||
[ladybug-type]
|
||||
(and (string? ladybug-type)
|
||||
(some? (re-matches #".+\[\d*\]$" ladybug-type))))
|
||||
|
||||
(defn struct-type?
|
||||
[ladybug-type]
|
||||
(and (string? ladybug-type)
|
||||
(str/starts-with? ladybug-type "STRUCT(")
|
||||
(not (list-type? ladybug-type))))
|
||||
|
||||
(declare format-typed-value)
|
||||
|
||||
(defn- format-typed-list
|
||||
"Cypher LIST literal, elements formatted by the element type.
|
||||
|
||||
Handles `T[]` and the fixed-size `T[n]` alike: the size constrains the column,
|
||||
not the literal."
|
||||
[ladybug-type v]
|
||||
(let [element (second (re-matches #"(.+?)\[\d*\]$" ladybug-type))
|
||||
elems (if (or (sequential? v) (set? v)) (seq v) [v])]
|
||||
(str "[" (str/join ", " (map #(format-typed-value element %) elems)) "]")))
|
||||
|
||||
(defn- format-struct
|
||||
"Cypher STRUCT literal, `{field: value, …}`.
|
||||
|
||||
*Every* declared field is emitted, NULL where the value has none: a struct
|
||||
literal's type is its field list, so omitting a field yields a different type
|
||||
and Ladybug refuses the implicit cast (`STRUCT(m2 DOUBLE, m4 DOUBLE)` cannot
|
||||
be assigned to `STRUCT(m1 …, m2 …, m3 …, m4 …)`). Penpot's layout margins are
|
||||
exactly that case — a shape sets only the sides it overrides."
|
||||
[ladybug-type v]
|
||||
(let [fields (values/struct-fields ladybug-type)]
|
||||
(str "{"
|
||||
(str/join ", "
|
||||
(for [[field field-type] fields
|
||||
:let [fv (get v field)]]
|
||||
;; Backticked for the same reason as in the DDL: a field
|
||||
;; named `column` is a keyword and will not parse bare.
|
||||
;; A bare NULL is typed STRING, which changes the struct's
|
||||
;; type as surely as omitting the field would, so absent
|
||||
;; fields get a NULL cast to their declared type.
|
||||
(str "`" field "`: "
|
||||
(if (nil? fv)
|
||||
(str "cast(NULL, '" field-type "')")
|
||||
(format-typed-value field-type fv)))))
|
||||
"}")))
|
||||
|
||||
(defn format-typed-value
|
||||
"Cypher literal for `v` in a column of `ladybug-type`.
|
||||
|
||||
Recursive over the type language, because the types are: a
|
||||
`MAP(UUID, STRUCT(…))` needs its keys, its fields and each field's own type
|
||||
honoured. `app.graph.schema.values/coerce` shapes the value first — turning a
|
||||
matrix record into six doubles, a hex colour into a packed integer — so this
|
||||
function only has to escape plain data.
|
||||
|
||||
`map-key-fn` renders the keys of a `MAP(STRING, …)`; the caller supplies it
|
||||
because the right form is a property of the column, not of this function
|
||||
(`app.graph.schema.contract/map-key-fn`)."
|
||||
([ladybug-type v] (format-typed-value ladybug-type v nil))
|
||||
([ladybug-type v map-key-fn]
|
||||
(let [v (values/coerce ladybug-type v)]
|
||||
(cond
|
||||
(nil? v)
|
||||
"NULL"
|
||||
|
||||
(list-type? ladybug-type)
|
||||
(format-typed-list ladybug-type v)
|
||||
|
||||
(map-type? ladybug-type)
|
||||
(let [[key-type value-type] (values/map-types ladybug-type)
|
||||
entries (seq v)
|
||||
format-key (if (and map-key-fn (= "STRING" key-type))
|
||||
#(format-string (map-key-fn (key %)))
|
||||
#(format-typed-value key-type (key %)))]
|
||||
(str "map([" (str/join ", " (map format-key entries))
|
||||
"], ["
|
||||
(str/join ", " (map #(format-typed-value value-type (val %)) entries))
|
||||
"])"))
|
||||
|
||||
(struct-type? ladybug-type)
|
||||
(format-struct ladybug-type v)
|
||||
|
||||
(= ladybug-type "JSON")
|
||||
(format-json v)
|
||||
|
||||
;; Coerce string ids from transit edge-cases into UUID literals.
|
||||
(= ladybug-type "UUID")
|
||||
(format-uuid v)
|
||||
|
||||
(= ladybug-type "TIMESTAMP")
|
||||
(format-timestamp v)
|
||||
|
||||
:else
|
||||
(format-value v)))))
|
||||
|
||||
(defn- ensure-semicolon
|
||||
[statement]
|
||||
(let [s (str/trim (str statement))]
|
||||
(if (str/ends-with? s ";") s (str s ";"))))
|
||||
|
||||
(defn- value->clj
|
||||
[^Value value]
|
||||
(when-not (.isNull value)
|
||||
(let [v (try
|
||||
(.getValue value)
|
||||
(catch Exception _
|
||||
;; LIST/STRUCT values are not supported by the binding's
|
||||
;; getValue (\"value_get_value\"); fall back to the textual
|
||||
;; representation so console queries do not crash.
|
||||
(.toString value)))]
|
||||
(cond
|
||||
(instance? Long v) v
|
||||
(instance? Integer v) (long v)
|
||||
(instance? Double v) v
|
||||
:else v))))
|
||||
|
||||
(defn- check-success!
|
||||
[^QueryResult result statement]
|
||||
(when-not (.isSuccess result)
|
||||
(let [err (.getErrorMessage result)]
|
||||
(ex/raise :type :internal
|
||||
:code :ladybug-query-failed
|
||||
:hint (str "Ladybug query failed: " err)
|
||||
:statement statement
|
||||
:err err))))
|
||||
|
||||
(defn- query-columns
|
||||
[^QueryResult result]
|
||||
(let [ncols (.getNumColumns result)]
|
||||
(vec (for [i (range ncols)]
|
||||
(.getColumnName result (long i))))))
|
||||
|
||||
(defn- query-row
|
||||
[^FlatTuple tuple ncols]
|
||||
(vec (for [i (range ncols)]
|
||||
(with-open [^Value value (.getValue tuple (long i))]
|
||||
(value->clj value)))))
|
||||
|
||||
(def ^:private default-query-max-rows 200)
|
||||
|
||||
(defn- read-query-rows
|
||||
[^QueryResult result ncols max-rows]
|
||||
(loop [rows [] n 0]
|
||||
(if (and (< n max-rows) (.hasNext result))
|
||||
(let [row (with-open [^FlatTuple tuple (.getNext result)]
|
||||
(query-row tuple ncols))]
|
||||
(recur (conj rows row) (inc n)))
|
||||
rows)))
|
||||
|
||||
(defn query-on-connection!
|
||||
"Execute a Cypher query on `conn` and return tabular results.
|
||||
|
||||
Returns `{:columns [...] :rows [[...] ...] :truncated? bool}`."
|
||||
[^Connection conn statement & {:keys [max-rows]
|
||||
:or {max-rows default-query-max-rows}}]
|
||||
(let [cypher (ensure-semicolon statement)]
|
||||
(with-open [^QueryResult result (.query conn cypher)]
|
||||
(check-success! result cypher)
|
||||
(let [ncols (long (.getNumColumns result))
|
||||
columns (query-columns result)
|
||||
rows (read-query-rows result ncols max-rows)
|
||||
total (long (.getNumTuples result))]
|
||||
{:columns columns
|
||||
:rows rows
|
||||
:truncated? (and (pos? total) (> total (count rows)))}))))
|
||||
|
||||
(def ^:private default-query-timeout-ms
|
||||
"0 disables query timeout (recommended for bulk COPY ingest)."
|
||||
0)
|
||||
|
||||
(defn- scalar-value
|
||||
[^Connection conn statement]
|
||||
(let [cypher (ensure-semicolon statement)]
|
||||
(with-open [^QueryResult result (.query conn cypher)]
|
||||
(check-success! result cypher)
|
||||
(when (.hasNext result)
|
||||
(with-open [^FlatTuple tuple (.getNext result)]
|
||||
(with-open [^Value value (.getValue tuple 0)]
|
||||
(value->clj value)))))))
|
||||
|
||||
(defn- extension-statement-ok?
|
||||
[err-msg]
|
||||
(let [err (str/lower-case (or err-msg ""))]
|
||||
(or (str/includes? err "already loaded")
|
||||
(str/includes? err "already installed"))))
|
||||
|
||||
(defn- run-extension-statement!
|
||||
[^Connection conn statement]
|
||||
(let [cypher (ensure-semicolon statement)]
|
||||
(with-open [^QueryResult result (.query conn cypher)]
|
||||
(when-not (.isSuccess result)
|
||||
(let [err (.getErrorMessage result)]
|
||||
(when-not (extension-statement-ok? err)
|
||||
(check-success! result cypher)))))))
|
||||
|
||||
(defn ensure-extensions!
|
||||
"Install and load Ladybug extensions required by graph ingest and sync."
|
||||
[^Connection conn]
|
||||
(run-extension-statement! conn "INSTALL json;")
|
||||
(run-extension-statement! conn "LOAD json;"))
|
||||
|
||||
(defn- run-statements!
|
||||
[^Connection conn statements]
|
||||
(doseq [statement statements]
|
||||
(let [cypher (ensure-semicolon statement)]
|
||||
(with-open [^QueryResult result (.query conn cypher)]
|
||||
(check-success! result cypher)))))
|
||||
|
||||
(defn- ensure-db-path!
|
||||
[db-path]
|
||||
(when-not (memory-db-path? db-path)
|
||||
(fs/create-dir (fs/parent db-path))))
|
||||
|
||||
(defn with-connection!
|
||||
"Open a Ladybug connection for `db-path` and invoke `(f conn)`.
|
||||
|
||||
Options:
|
||||
- `:query-timeout-ms` query timeout in milliseconds (default 0, disabled)
|
||||
|
||||
For `:memory:`, the database only lives for the duration of this call;
|
||||
all reads and writes must happen inside `f`."
|
||||
[db-path f & {:keys [query-timeout-ms]
|
||||
:or {query-timeout-ms default-query-timeout-ms}}]
|
||||
(ensure-db-path! db-path)
|
||||
(let [^Database db (if (memory-db-path? db-path)
|
||||
(Database.)
|
||||
(Database. (str db-path)))]
|
||||
(try
|
||||
(let [^Connection conn (Connection. db)]
|
||||
(try
|
||||
(.setQueryTimeout conn (long query-timeout-ms))
|
||||
(ensure-extensions! conn)
|
||||
(f conn)
|
||||
(finally
|
||||
(.close conn))))
|
||||
(finally
|
||||
(.close db)))))
|
||||
|
||||
(defn exec-on-connection!
|
||||
"Execute Cypher statements on an open Ladybug connection."
|
||||
[^Connection conn statements]
|
||||
(assert (sequential? statements) "statements should be a sequential collection")
|
||||
(run-statements! conn statements))
|
||||
|
||||
;; --- prepared statements
|
||||
|
||||
(defn- ->param-value
|
||||
"Clojure scalar → `Value` for prepared-statement binding.
|
||||
|
||||
This is the only `Value` constructor on the write path, so every parameter
|
||||
is wrapped here. Parameters are scalars: the `Value` constructor takes no
|
||||
list or map, so `MAP`, `STRUCT` and `T[]` columns stay literal-rendered
|
||||
(`format-typed-value`) and the `:else` raise below means a caller tried to
|
||||
bind one."
|
||||
^Value [v]
|
||||
(cond
|
||||
(nil? v) (Value/createNull) ; no explicit type needed
|
||||
(uuid? v) (Value. ^Object v) ; native UUID
|
||||
(string? v) (Value. ^Object v)
|
||||
(boolean? v) (Value. ^Object v)
|
||||
(integer? v) (Value. ^Object (long v))
|
||||
(number? v) (Value. ^Object (double v))
|
||||
(keyword? v) (Value. ^Object (name v))
|
||||
|
||||
(instance? java.time.Instant v) ; native TIMESTAMP
|
||||
(Value. ^Object v)
|
||||
|
||||
(instance? java.util.Date v)
|
||||
(Value. ^Object (.toInstant ^java.util.Date v))
|
||||
|
||||
:else
|
||||
(ex/raise :type :internal
|
||||
:code :ladybug-unsupported-param
|
||||
:hint (str "cannot bind a " (type v) " as a Ladybug parameter; "
|
||||
"compound columns must be literal-rendered")
|
||||
:value v)))
|
||||
|
||||
(defn- as-statement
|
||||
"Normalize a statement to `{:cypher … :params …}`.
|
||||
|
||||
A bare string binds nothing, so the sync builders can convert to bound
|
||||
parameters one family at a time."
|
||||
[stmt]
|
||||
(if (map? stmt)
|
||||
(update stmt :params #(or % {}))
|
||||
{:cypher stmt :params {}}))
|
||||
|
||||
(defn prepare-on-connection!
|
||||
"Parse and bind `statement` on `conn` without executing it.
|
||||
|
||||
The returned `PreparedStatement` is a JNI resource: the caller closes it."
|
||||
^PreparedStatement [^Connection conn statement]
|
||||
(let [cypher (ensure-semicolon statement)
|
||||
ps (.prepare conn cypher)]
|
||||
(when-not (.isSuccess ps)
|
||||
(let [err (.getErrorMessage ps)]
|
||||
(.close ps)
|
||||
(ex/raise :type :internal
|
||||
:code :ladybug-prepare-failed
|
||||
:hint (str "Ladybug prepare failed: " err)
|
||||
:statement cypher
|
||||
:err err)))
|
||||
ps))
|
||||
|
||||
(defn execute-prepared!
|
||||
"Bind `params` into `ps` and execute it on `conn`.
|
||||
|
||||
`params` keys are parameter names without the `$` (keyword or string);
|
||||
values are scalars. Every bound `Value` is closed, including the ones built
|
||||
before a later parameter is rejected."
|
||||
[^Connection conn ^PreparedStatement ps params]
|
||||
(let [vmap (java.util.HashMap.)]
|
||||
(try
|
||||
(doseq [[k v] params]
|
||||
(.put vmap (name k) (->param-value v)))
|
||||
(with-open [^QueryResult result (.execute conn ps vmap)]
|
||||
(check-success! result "<prepared>"))
|
||||
(finally
|
||||
(run! #(.close ^Value %) (.values vmap))))))
|
||||
|
||||
(defn exec-prepared-on-connection!
|
||||
"Prepare all statements, then execute all of them.
|
||||
|
||||
A parse or bind failure in *any* statement aborts the batch before the first
|
||||
mutation runs — the bind-level batch gate. Statements are
|
||||
`{:cypher … :params {…}}` maps or bare strings."
|
||||
[^Connection conn stmts]
|
||||
(assert (sequential? stmts) "statements should be a sequential collection")
|
||||
(let [prepared (volatile! [])]
|
||||
(try
|
||||
(doseq [stmt stmts]
|
||||
(let [{:keys [cypher params]} (as-statement stmt)]
|
||||
(vswap! prepared conj {:ps (prepare-on-connection! conn cypher)
|
||||
:params params})))
|
||||
(doseq [{:keys [ps params]} @prepared]
|
||||
(execute-prepared! conn ps params))
|
||||
(finally
|
||||
(run! #(.close ^PreparedStatement (:ps %)) @prepared)))))
|
||||
|
||||
(defn validate-on-connection!
|
||||
"Binder gate: parse and semantic-check `statement` against the live schema,
|
||||
without executing it.
|
||||
|
||||
Returns `{:ok? … :error … :read-only? …}`. Unlike `prepare-on-connection!`
|
||||
a failure is a return value rather than a raise: the callers are gates (the
|
||||
CI binder gate, the console read-only gate) that report it. `:read-only?` is
|
||||
the engine's own read/write analysis."
|
||||
[^Connection conn statement]
|
||||
(with-open [^PreparedStatement ps (.prepare conn (ensure-semicolon statement))]
|
||||
(let [ok? (.isSuccess ps)]
|
||||
{:ok? ok?
|
||||
:error (when-not ok? (.getErrorMessage ps))
|
||||
:read-only? (when ok? (.isReadOnly ps))})))
|
||||
|
||||
(defn query-scalar-on-connection!
|
||||
"Execute a query expected to return a single scalar value on `conn`."
|
||||
[^Connection conn statement]
|
||||
(scalar-value conn statement))
|
||||
|
||||
(defn exec!
|
||||
"Execute Cypher statements against a Ladybug database.
|
||||
|
||||
`db-path` is either `:memory:` or a filesystem path to a `.lbug` database."
|
||||
[db-path statements]
|
||||
(with-connection! db-path
|
||||
(fn [conn]
|
||||
(exec-on-connection! conn statements))))
|
||||
|
||||
(defn query-scalar!
|
||||
"Execute a query expected to return a single scalar value."
|
||||
[db-path statement]
|
||||
(with-connection! db-path
|
||||
(fn [conn]
|
||||
(query-scalar-on-connection! conn statement))))
|
||||
|
||||
(defn smoke-test!
|
||||
"Run a minimal CREATE + count against Ladybug."
|
||||
[& {:keys [db-path] :or {db-path ":memory:"}}]
|
||||
(when-not (memory-db-path? db-path)
|
||||
(reset-db-path! db-path))
|
||||
(with-connection! db-path
|
||||
(fn [^Connection conn]
|
||||
(run-statements! conn
|
||||
["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));"
|
||||
"CREATE (:Person {name: 'Alice', age: 25});"
|
||||
"CREATE (:Person {name: 'Bob', age: 30});"])
|
||||
{:db-path db-path
|
||||
:person-count (scalar-value conn
|
||||
"MATCH (a:Person) RETURN count(a) AS c;")})))
|
||||
@@ -0,0 +1,59 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.meta
|
||||
"`GraphMeta`: the graph's own account of who built it and from what.
|
||||
|
||||
A projected graph is a cache of a file at a revision, built by a known
|
||||
schema. The row records both, so a reader can decide whether to reuse the
|
||||
database or rebuild it: a `schema_version` that no longer matches the
|
||||
registry, or a `source_revn` behind the file's, means the cache is stale.
|
||||
|
||||
The row is written *last* in a build, so its presence also marks the build
|
||||
complete.
|
||||
|
||||
Keyed by `source_file_id` rather than holding a single row: a closure graph
|
||||
is a union of per-file builds, and each contributing file keeps its own
|
||||
provenance."
|
||||
(:require
|
||||
[app.common.time :as ct]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes])
|
||||
(:import
|
||||
com.ladybugdb.Connection))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def table
|
||||
"GraphMeta")
|
||||
|
||||
(def producer
|
||||
"penpot")
|
||||
|
||||
(def ddl
|
||||
"DDL for the provenance table."
|
||||
(str "CREATE NODE TABLE `" table "` ("
|
||||
"`source_file_id` UUID, "
|
||||
"`producer` STRING, "
|
||||
"`producer_version` STRING, "
|
||||
"`schema_version` STRING, "
|
||||
"`source_revn` INT64, "
|
||||
"`built_at` TIMESTAMP, "
|
||||
"PRIMARY KEY (`source_file_id`));"))
|
||||
|
||||
(defn write!
|
||||
"Record what this build produced for `file-id`."
|
||||
[^Connection conn {:keys [file-id revn]}]
|
||||
(ladybug/exec-on-connection! conn [ddl])
|
||||
(ladybug/exec-on-connection!
|
||||
conn
|
||||
[(str "MERGE (m:`" table "` {source_file_id: " (ladybug/format-uuid file-id) "}) "
|
||||
"SET m.producer = " (ladybug/format-string producer) ", "
|
||||
"m.producer_version = " (ladybug/format-string (or (System/getenv "PENPOT_BUILD") "devenv")) ", "
|
||||
"m.schema_version = " (ladybug/format-string nodes/schema-version) ", "
|
||||
"m.source_revn = " (ladybug/format-int (or revn 0)) ", "
|
||||
"m.built_at = " (ladybug/format-timestamp (ct/now)) ";")]))
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.projection.document
|
||||
"Project a Penpot file-data map into Ladybug nodes and structural edges.
|
||||
|
||||
Projects Document, Page, Component, the full shape tree (skipping the root
|
||||
frame), and `IsChildOf` edges from shapes/pages/components to their parent.
|
||||
|
||||
Two denormalizations happen here rather than in a later pass, because the
|
||||
walk already has both answers in hand and a post-ingest statement would have
|
||||
to rediscover them:
|
||||
|
||||
- `page-id` on every shape, from the page the walk is currently in;
|
||||
- `component-id` propagated from an instance head down to its descendants,
|
||||
from the head context the walk carries."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.graph.schema.nodes :as nodes]))
|
||||
|
||||
(def root-frame-id
|
||||
uuid/zero)
|
||||
|
||||
(defn- document-attrs
|
||||
"The Document node's attrs: the file row, minus its data blob.
|
||||
|
||||
`:options` is lifted out of the blob before it goes: it is file-level
|
||||
configuration a consumer wants without opening `:data`."
|
||||
[file data]
|
||||
(-> file
|
||||
(assoc :id (or (:id data) (:id file)))
|
||||
(cond-> (:options data) (assoc :options (:options data)))
|
||||
(dissoc :data)))
|
||||
|
||||
(defn- page-attrs
|
||||
[page index]
|
||||
(-> page
|
||||
(dissoc :objects)
|
||||
(cond-> (some? index) (assoc :index (long index)))))
|
||||
|
||||
(defn- component-attrs
|
||||
[component]
|
||||
(-> component
|
||||
(dissoc :objects)
|
||||
;; schema:component requires :path; some legacy rows omit it
|
||||
(update :path #(or % ""))))
|
||||
|
||||
(defn- shape-table
|
||||
[shape]
|
||||
(nodes/table-for-type (:type shape)))
|
||||
|
||||
(defn denormalized-shape
|
||||
"`shape` with `page-id` set and an inherited `component-id` filled in.
|
||||
|
||||
A shape that carries its own `component-id` keeps it; `component-ctx` only
|
||||
fills the gap for descendants (see `descend-component-ctx`)."
|
||||
[shape page-id component-ctx]
|
||||
(cond-> (assoc shape :page-id page-id)
|
||||
(and (uuid? component-ctx) (nil? (:component-id shape)))
|
||||
(assoc :component-id component-ctx)))
|
||||
|
||||
(defn- shape-node-attrs
|
||||
[table shape page-id component-ctx]
|
||||
(nodes/project-attrs table (denormalized-shape shape page-id component-ctx)))
|
||||
|
||||
(defn descend-component-ctx
|
||||
"The component context to pass to `shape`'s children.
|
||||
|
||||
Inheritance stops at the nearest ancestor Frame carrying a `component-id`,
|
||||
and any intermediate shape that carries one is a barrier:
|
||||
|
||||
- a Frame with its own `component-id` becomes the new context (it is an
|
||||
instance head, and its descendants belong to *it*, not to an outer head);
|
||||
- any other shape carrying a `component-id` blocks inheritance below it
|
||||
without being able to supply one, since only Frames are heads;
|
||||
- otherwise the context passes through unchanged."
|
||||
[table shape ctx]
|
||||
(let [own (:component-id shape)]
|
||||
(cond
|
||||
(and (some? own) (= table "Frame")) own
|
||||
(some? own) ::blocked
|
||||
:else ctx)))
|
||||
|
||||
(defn- container-table?
|
||||
[table]
|
||||
(contains? nodes/container-tables table))
|
||||
|
||||
(defn- child-shape-ids
|
||||
"Child ids in Penpot z-order (reversed from the stored :shapes list)."
|
||||
[parent]
|
||||
(when-let [shapes (:shapes parent)]
|
||||
(vec (reverse shapes))))
|
||||
|
||||
(defn- initial-acc
|
||||
[]
|
||||
{:nodes {}
|
||||
:edges []
|
||||
:stats {:documents 0 :pages 0 :components 0 :shapes 0}})
|
||||
|
||||
(declare project-shape-ids)
|
||||
|
||||
(defn- project-shape
|
||||
[objects acc table shape parent-table parent-id position page-id component-ctx]
|
||||
(let [shape-id (:id shape)
|
||||
acc' (-> acc
|
||||
(update-in [:nodes table] (fnil conj [])
|
||||
(shape-node-attrs table shape page-id component-ctx))
|
||||
(update :edges conj {:from-table table
|
||||
:from-id shape-id
|
||||
:to-table parent-table
|
||||
:to-id parent-id
|
||||
:position position})
|
||||
(update-in [:stats :shapes] inc))]
|
||||
(if-let [child-ids (when (container-table? table)
|
||||
(child-shape-ids shape))]
|
||||
(project-shape-ids objects acc' table shape-id child-ids page-id
|
||||
(descend-component-ctx table shape component-ctx))
|
||||
acc')))
|
||||
|
||||
(defn- project-shape-ids
|
||||
[objects acc parent-table parent-id child-ids page-id component-ctx]
|
||||
(reduce
|
||||
(fn [acc [position shape-id]]
|
||||
(if-let [shape (get objects shape-id)]
|
||||
(if-let [table (shape-table shape)]
|
||||
(project-shape objects acc table shape parent-table parent-id position
|
||||
page-id component-ctx)
|
||||
(do
|
||||
(l/wrn :hint "unsupported shape type for graph slice"
|
||||
:shape-id (str shape-id)
|
||||
:type (:type shape))
|
||||
acc))
|
||||
(do
|
||||
(l/wrn :hint "missing shape in page objects"
|
||||
:shape-id (str shape-id))
|
||||
acc)))
|
||||
acc
|
||||
(map-indexed vector child-ids)))
|
||||
|
||||
(defn- project-page
|
||||
[acc doc-id page position]
|
||||
(let [page-id (:id page)
|
||||
objects (:objects page)
|
||||
root (get objects root-frame-id)
|
||||
page-node (nodes/project-attrs "Page" (page-attrs page position))
|
||||
acc' (-> acc
|
||||
(update-in [:nodes "Page"] (fnil conj []) page-node)
|
||||
(update :edges conj {:from-table "Page"
|
||||
:from-id page-id
|
||||
:to-table "Document"
|
||||
:to-id doc-id
|
||||
:position position})
|
||||
(update-in [:stats :pages] inc))]
|
||||
(if-let [top-level-ids (child-shape-ids root)]
|
||||
(project-shape-ids objects acc' "Page" page-id top-level-ids page-id nil)
|
||||
acc')))
|
||||
|
||||
(defn- project-component
|
||||
[acc doc-id component position]
|
||||
(if (:deleted component)
|
||||
acc
|
||||
(let [comp-id (:id component)
|
||||
node (nodes/project-attrs "Component" (component-attrs component))]
|
||||
(-> acc
|
||||
(update-in [:nodes "Component"] (fnil conj []) node)
|
||||
(update :edges conj {:from-table "Component"
|
||||
:from-id comp-id
|
||||
:to-table "Document"
|
||||
:to-id doc-id
|
||||
:position position})
|
||||
(update-in [:stats :components] inc)))))
|
||||
|
||||
(defn- project-components
|
||||
[acc doc-id components]
|
||||
(reduce (fn [acc [position [_id component]]]
|
||||
(project-component acc doc-id component position))
|
||||
acc
|
||||
(map-indexed vector components)))
|
||||
|
||||
(defn projection-data
|
||||
"Build node/edge rows for projecting `data` into Ladybug.
|
||||
|
||||
Returns `{:nodes {table [attrs ...]} :edges [...] :stats {...}}`."
|
||||
[data file]
|
||||
(let [doc-id (or (:id data) (:id file))
|
||||
doc-node (nodes/project-attrs "Document" (document-attrs file data))
|
||||
;; `:pages` is the tab order the user sees, and `Page.index` and the
|
||||
;; page's `IsChildOf.position` are that order. Child shapes are
|
||||
;; reversed on the way in (`child-shape-ids`) because their stored
|
||||
;; list runs bottom to top; pages have no such second ordering.
|
||||
pages (seq (:pages data))
|
||||
comps (seq (:components data))
|
||||
acc0 (-> (initial-acc)
|
||||
(update-in [:nodes "Document"] (fnil conj []) doc-node)
|
||||
(assoc-in [:stats :documents] 1))
|
||||
acc (cond-> acc0
|
||||
(seq comps)
|
||||
(project-components doc-id comps))
|
||||
acc (if (empty? pages)
|
||||
acc
|
||||
(reduce (fn [acc [position page-id]]
|
||||
(if-let [page (get-in data [:pages-index page-id])]
|
||||
(project-page acc doc-id page position)
|
||||
(do
|
||||
(l/wrn :hint "missing page in pages-index"
|
||||
:page-id (str page-id))
|
||||
acc)))
|
||||
acc
|
||||
(map-indexed vector pages)))]
|
||||
(select-keys acc [:nodes :edges :stats])))
|
||||
@@ -0,0 +1,149 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.projection.transforms
|
||||
"Derived graph links: edges a reader could compute from the projected
|
||||
columns, materialized once at build time so a query does not have to.
|
||||
|
||||
Each entry in `registry` names the transform, the relationship it produces,
|
||||
and the function that produces it, so adding one is a single entry and
|
||||
nothing else has to be told about it."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes])
|
||||
(:import
|
||||
com.ladybugdb.Connection))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- run-scalar!
|
||||
[^Connection conn statement]
|
||||
(or (ladybug/query-scalar-on-connection! conn statement) 0))
|
||||
|
||||
(defn- link-component-instances!
|
||||
"`IsInstanceOf` from Frame instance heads to their Component.
|
||||
|
||||
Every head is linked, the main instance and any copy root alike.
|
||||
|
||||
`component-file` is what makes a head a head here, not `component-id` alone.
|
||||
`app.common.types.component/instance-of?` requires both, and the projection
|
||||
denormalizes `component-id` down the shape tree
|
||||
(`app.graph.projection.document`), so on its own it no longer distinguishes a
|
||||
head from a shape that merely lives inside one. `component-file` is not
|
||||
denormalized and remains the head marker Penpot itself uses."
|
||||
[^Connection conn]
|
||||
(run-scalar! conn
|
||||
(str "MATCH (f:Frame), (c:Component) "
|
||||
"WHERE f.component_id = c.id "
|
||||
"AND f.component_file IS NOT NULL "
|
||||
"AND NOT COALESCE(c.deleted, false) "
|
||||
"MERGE (f)-[:IsInstanceOf]->(c) "
|
||||
"RETURN count(*);")))
|
||||
|
||||
(defn- shape-pair-statements
|
||||
"One statement per (from, to) shape-table pair.
|
||||
|
||||
Ladybug cannot create a relationship bound by multiple node labels in a
|
||||
single `MERGE`, a constraint inherited from Kùzu, which it forks (upstream
|
||||
issue kuzudb/kuzu#5841). The loop over label pairs is that dialect
|
||||
constraint, not a modelling choice."
|
||||
[f]
|
||||
(for [from nodes/shape-tables
|
||||
to nodes/shape-tables]
|
||||
(f from to)))
|
||||
|
||||
(defn- link-shape-refs!
|
||||
"`RefersTo` from an instance shape to its homologue in the main instance,
|
||||
driven by `shape-ref`."
|
||||
[^Connection conn]
|
||||
(reduce
|
||||
(fn [total statement] (+ total (run-scalar! conn statement)))
|
||||
0
|
||||
(shape-pair-statements
|
||||
(fn [from to]
|
||||
(str "MATCH (s:" (nodes/match-label from) "), (t:" (nodes/match-label to) ") "
|
||||
"WHERE s.shape_ref = t.id "
|
||||
"MERGE (s)-[:RefersTo]->(t) "
|
||||
"RETURN count(*);")))))
|
||||
|
||||
(def ^:private swap-slot-prefix "swap-slot-")
|
||||
|
||||
(def ^:private slot-uuid-expr
|
||||
;; Ladybug `substring` is 1-indexed; 36 = RFC 4122 UUID text length.
|
||||
(str "substring(touched_key, " (inc (count swap-slot-prefix)) ", 36)"))
|
||||
|
||||
(defn- link-swap-slots!
|
||||
"`FillsSwapSlot` from a swapped-in shape to the slot it replaces.
|
||||
|
||||
Penpot records a component sub-shape swap as a `swap-slot-<uuid>` entry in
|
||||
the *replacing* shape's `touched` set, where `<uuid>` names the replaced
|
||||
slot shape in the main instance. The entries are then stripped from
|
||||
`touched`, as `app.common.types.component/normal-touched-groups` does, so a
|
||||
reader of `touched` sees design edits rather than swap bookkeeping.
|
||||
|
||||
Stripping makes this the one transform that writes a column another
|
||||
transform could read. Anything reading `touched` has to run before it."
|
||||
[^Connection conn]
|
||||
(let [linked
|
||||
(reduce
|
||||
(fn [total statement] (+ total (run-scalar! conn statement)))
|
||||
0
|
||||
(shape-pair-statements
|
||||
(fn [from to]
|
||||
(str "MATCH (s:" (nodes/match-label from) ") "
|
||||
"WHERE size(s.touched) > 0 "
|
||||
"UNWIND s.touched AS touched_key "
|
||||
"WITH s, touched_key "
|
||||
"WHERE STARTS_WITH(touched_key, '" swap-slot-prefix "') "
|
||||
"WITH s, CAST(" slot-uuid-expr ", 'UUID') AS slot_id "
|
||||
"MATCH (t:" (nodes/match-label to) ") "
|
||||
"WHERE t.id = slot_id AND s.id <> t.id "
|
||||
"MERGE (s)-[r:FillsSwapSlot {slot_id: slot_id}]->(t) "
|
||||
"RETURN count(r);"))))]
|
||||
;; Strip unconditionally: an entry may name a slot that was garbage
|
||||
;; collected, so "no edge created" does not mean "nothing to strip".
|
||||
(doseq [table nodes/shape-tables]
|
||||
(ladybug/exec-on-connection!
|
||||
conn
|
||||
[(str "MATCH (s:" (nodes/match-label table) ") "
|
||||
"WHERE size(s.touched) > 0 "
|
||||
"SET s.touched = list_filter(s.touched, x -> "
|
||||
"NOT STARTS_WITH(x, '" swap-slot-prefix "'));")]))
|
||||
linked))
|
||||
|
||||
(def registry
|
||||
"Every transform this backend applies.
|
||||
|
||||
`:id` names the transform in the ingest report and the log. `:rel` names
|
||||
the relationship it produces. The three registered here read disjoint
|
||||
columns, so the vector order is not load-bearing. The one ordering
|
||||
constraint that exists is stated on `link-swap-slots!`."
|
||||
[{:id "link-component-instances" :rel :IsInstanceOf :run link-component-instances!}
|
||||
{:id "link-shape-refs" :rel :RefersTo :run link-shape-refs!}
|
||||
{:id "link-swap-slots" :rel :FillsSwapSlot :run link-swap-slots!}])
|
||||
|
||||
(defn apply-transforms!
|
||||
"Apply every registered transform to an already loaded graph.
|
||||
|
||||
Returns `{:ids [...] :counts {...} :transforms n}`, where `:ids` names what
|
||||
ran and `:counts` gives the edges each one produced."
|
||||
[_system ^Connection conn _data _file]
|
||||
(reduce
|
||||
(fn [acc {:keys [id rel run]}]
|
||||
(let [n (run conn)]
|
||||
(l/inf :hint "graph transform" :transform id :edges n)
|
||||
(-> acc
|
||||
(update :ids conj id)
|
||||
(update :counts assoc rel n)
|
||||
(assoc rel n))))
|
||||
{:ids [] :counts {} :transforms (count registry)}
|
||||
registry))
|
||||
|
||||
(defn transform-ids
|
||||
"Ids of every transform in the registry."
|
||||
[]
|
||||
(mapv :id registry))
|
||||
@@ -0,0 +1,65 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.report
|
||||
(:require
|
||||
[clojure.core :as c]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn- println!
|
||||
[& lines]
|
||||
(doseq [line lines]
|
||||
(println line)))
|
||||
|
||||
(defn- section-title
|
||||
[title]
|
||||
(println! (str "\n" title)
|
||||
(str (apply str (repeat (count title) "─")))))
|
||||
|
||||
(defn- kv-line
|
||||
[k v]
|
||||
(format " %-14s %s" (str k ":") v))
|
||||
|
||||
(defn- print-node-counts
|
||||
[nodes]
|
||||
(doseq [[table count] (sort-by first nodes)
|
||||
:when (pos? (long count))]
|
||||
(println! (kv-line table count))))
|
||||
|
||||
(defn print-ingest!
|
||||
"Pretty-print the result map returned by `app.graph.ingest/ingest-file!`."
|
||||
[{:keys [file-id revn name db-path schema-version projection transforms stats]}]
|
||||
(section-title "Graph ingest")
|
||||
(println! (kv-line "File" (str name " (" file-id ")"))
|
||||
(kv-line "Revision" revn)
|
||||
(kv-line "Schema" schema-version)
|
||||
(kv-line "Database" db-path))
|
||||
|
||||
(when-let [pstats (:stats projection)]
|
||||
(section-title "Projection")
|
||||
(doseq [[k v] (sort-by key pstats)]
|
||||
(println! (kv-line (c/name k) v))))
|
||||
|
||||
(section-title "Transforms")
|
||||
(println! (kv-line "Applied" (or (:transforms transforms) 0)))
|
||||
(doseq [[rel count] (sort-by key (:counts transforms))]
|
||||
(println! (kv-line (c/name rel) count)))
|
||||
(when-let [ids (seq (:ids transforms))]
|
||||
(println! (kv-line "Recorded" (str/join ", " ids))))
|
||||
|
||||
(when stats
|
||||
(section-title "Graph counts")
|
||||
(when-let [nodes (:nodes stats)]
|
||||
(println! " Nodes")
|
||||
(print-node-counts nodes))
|
||||
(when-let [edges (:edges stats)]
|
||||
(println! " Edges")
|
||||
(doseq [[rel count] (sort-by key edges)
|
||||
:when (pos? (long count))]
|
||||
(println! (kv-line (c/name rel) count)))))
|
||||
|
||||
(println!)
|
||||
nil)
|
||||
@@ -0,0 +1,30 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema
|
||||
"Ladybug DDL facade for the graph-backed Penpot vertical slice.
|
||||
|
||||
Node metadata and DDL generation live in `app.graph.schema.nodes`."
|
||||
(:require
|
||||
[app.graph.schema.nodes :as nodes]))
|
||||
|
||||
(def schema-version
|
||||
nodes/schema-version)
|
||||
|
||||
(def container-node-tables
|
||||
nodes/container-tables)
|
||||
|
||||
(def shape-node-tables
|
||||
nodes/shape-tables)
|
||||
|
||||
(def node-tables
|
||||
(mapv (fn [{:keys [table schema]}]
|
||||
{:name table :schema schema})
|
||||
nodes/node-types))
|
||||
|
||||
(defn ddl-statements
|
||||
[]
|
||||
(nodes/ddl-statements))
|
||||
@@ -0,0 +1,150 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema.contract
|
||||
"Deliberate choices in Penpot's graph schema, recorded as data.
|
||||
|
||||
Penpot must pick a spelling and a type for every graph column. A Ladybug
|
||||
column gets both once, at table creation, and neither widens afterwards. The
|
||||
choices are therefore worth making deliberately and worth recording.
|
||||
|
||||
Three of them live here:
|
||||
|
||||
- `column-name` maps a Penpot key to its column. The rule is snake_case of
|
||||
the key, and `renames` records every exception.
|
||||
- `dropped-keys` and `per-table-dropped` name Penpot keys that deliberately
|
||||
get no column.
|
||||
- `type-overrides` pins the Ladybug type where the Malli-derived one
|
||||
(`app.graph.schema.types`) is coarser than the column deserves.
|
||||
|
||||
Each entry carries its reason. A divergence from the default rule is then a
|
||||
diff to review rather than a silent rename."
|
||||
(:require
|
||||
[app.common.json :as json]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(def ^:private renames
|
||||
"Penpot key to column name, where the column is not snake_case of the key.
|
||||
|
||||
Keyed by the Penpot key alone: no shape type gives one of these a second
|
||||
meaning, so a per-table map would only add ceremony."
|
||||
{;; `bool` collides with the Ladybug type name, so the column is named after
|
||||
;; the table (`Boolean`) rather than after Penpot's `:bool` shape type.
|
||||
:bool-type "boolean_type"
|
||||
|
||||
;; The column records what the file saved, which can lag what the shape
|
||||
;; tree implies. The `saved_` prefix marks it as the stored value rather
|
||||
;; than a derivation.
|
||||
:component-root "saved_component_root"
|
||||
|
||||
;; The value is a list, so the plural is accurate.
|
||||
:shadow "shadows"
|
||||
|
||||
;; The column spells the revision number out.
|
||||
:revn "revision"})
|
||||
|
||||
(def dropped-keys
|
||||
"Penpot keys projected by the Malli registry that get no column.
|
||||
|
||||
Dropping is right only when the column would be dead weight for every reader
|
||||
of the graph. A key a reader might learn from belongs in `unprojected-keys`
|
||||
instead."
|
||||
{:deleted-at
|
||||
"Only non-nil for a soft-deleted file, and a deleted file is never ingested."
|
||||
|
||||
:pixel-grid-color
|
||||
"Viewer chrome: the color of the editor's pixel grid, not design content."
|
||||
|
||||
:pixel-grid-opacity
|
||||
"Viewer chrome, as above."})
|
||||
|
||||
(def unprojected-keys
|
||||
"Penpot keys that should become graph columns and do not have one yet.
|
||||
|
||||
Distinct from `dropped-keys` on purpose: these are a debt the projection
|
||||
owes, not a decision to discard data. Keeping the two apart means a new
|
||||
upstream attribute cannot be quietly buried in the drop list."
|
||||
{:background-blur
|
||||
"Landed upstream behind a default-on flag. No column for it yet."})
|
||||
|
||||
(def ^:private per-table-dropped
|
||||
"Keys dropped only on certain tables.
|
||||
|
||||
`:grids` is the standing case: Penpot's shape schema admits it on every
|
||||
shape, but only a Frame ever carries one. Emitting an always-null column on
|
||||
ten other tables would widen every multi-table scan for nothing."
|
||||
{:grids #{"Boolean" "Circle" "Group" "Image" "Path" "Rectangle" "SVGRaw" "Text"}})
|
||||
|
||||
(def type-overrides
|
||||
"Ladybug column type per column name, where the derived type is too coarse.
|
||||
|
||||
`app.graph.schema.types` derives a type from the Malli schema, which is the
|
||||
right default but coarser than the column deserves in places: a Malli `:map`
|
||||
becomes `JSON`, where a native Ladybug MAP or a fixed-size array lets a
|
||||
consumer read a tensor row without parsing.
|
||||
|
||||
Only load-bearing divergences are pinned here, in the order they became
|
||||
load-bearing."
|
||||
{;; Must be a native MAP: a JSON blob cannot be indexed by key in Cypher, so
|
||||
;; `map_keys` and `map_extract` cannot reach a single token at all.
|
||||
"applied_tokens" "MAP(STRING, STRING)"
|
||||
|
||||
;; `grc/schema:rect` is an inline `:and` over a map, not the registered
|
||||
;; `::grc/rect`, so `app.graph.schema.types` cannot recognize it by type.
|
||||
;; Four doubles rather than the eight-field struct: `x1`/`y1`/`x2`/`y2` are
|
||||
;; derivable from `x`/`y`/`width`/`height`, and a fixed-size array is a
|
||||
;; tensor row a consumer reads without parsing.
|
||||
"selrect" "DOUBLE[4]"
|
||||
|
||||
;; The SVG provenance attributes are typed `:map` in the shape schema on
|
||||
;; purpose. Legacy files hold them as plain maps rather than as
|
||||
;; `::grc/rect` and `::gmt/matrix` records, and a tighter *schema* would
|
||||
;; reject those files
|
||||
;; (`app.common.types.shape/schema:shape-generic-attrs`). A tighter
|
||||
;; *column* is free: `app.graph.schema.values/coerce` reads either form.
|
||||
"svg_viewbox" "DOUBLE[4]"
|
||||
"svg_transform" "DOUBLE[6]"
|
||||
|
||||
;; `:fills` is an `:or` over the packed `app.common.types.fills` value and
|
||||
;; a plain vector of fill maps, so the schema alone cannot say it is a
|
||||
;; collection. It always is one, and a fill has enough optional shape
|
||||
;; (solid, gradient, image) that JSON per element is the honest element
|
||||
;; type.
|
||||
"fills" "JSON[]"})
|
||||
|
||||
(def ^:private map-key-fns
|
||||
"How to render the *keys* of a MAP column, per column.
|
||||
|
||||
A column name is schema, so it is snake_case. The keys inside a MAP are
|
||||
values, so they keep the spelling their producer used. `applied_tokens` is
|
||||
keyed by shape attribute in the camelCase form
|
||||
`app.common.json/write-camel-key` produces: `strokeWidth`, not
|
||||
`stroke-width`."
|
||||
{"applied_tokens" json/write-camel-key})
|
||||
|
||||
(defn map-key-fn
|
||||
"Key renderer for a MAP column. `name` unless the column says otherwise."
|
||||
[column]
|
||||
(get map-key-fns column name))
|
||||
|
||||
(defn column-name
|
||||
"The graph column name for Penpot key `k`.
|
||||
|
||||
Default: snake_case of the key. `renames` overrides."
|
||||
[k]
|
||||
(or (get renames k)
|
||||
(str/replace (name k) "-" "_")))
|
||||
|
||||
(defn drop-key?
|
||||
"Should key `k` be omitted from `table`'s columns?"
|
||||
[table k]
|
||||
(or (contains? dropped-keys k)
|
||||
(contains? (get per-table-dropped k #{}) table)))
|
||||
|
||||
(defn ladybug-type
|
||||
"The pinned Ladybug type for `column`, or `fallback` when nothing is pinned."
|
||||
[column fallback]
|
||||
(get type-overrides column fallback))
|
||||
@@ -0,0 +1,343 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema.nodes
|
||||
"Single source of truth for graph node tables.
|
||||
|
||||
Each registry entry declares Penpot Malli sources plus projection
|
||||
options (`:drop`, optional `:extra`). Derived artifacts — Ladybug
|
||||
DDL, Arrow fields, validation, type dispatch — all flow from that.
|
||||
|
||||
This registry is the single source of the graph schema. A Ladybug column
|
||||
gets its name and its type once, at table creation, and there is no
|
||||
widening afterwards. Every divergence between a Penpot key and its column
|
||||
is recorded in `app.graph.schema.contract`."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.component :as ctk]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.page :as ctp]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.contract :as contract]
|
||||
[app.graph.schema.projection :as projection]
|
||||
[app.graph.schema.types :as types]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(def schema-version
|
||||
"penpot-graph-slice-4")
|
||||
|
||||
(def ^:private document-projection
|
||||
{:source ctf/schema:file
|
||||
:drop [:data]
|
||||
;; Attributes a file map carries that `ctf/schema:file` does not declare.
|
||||
;;
|
||||
;; They belong here rather than in that schema, even though the graph wants
|
||||
;; them, because `schema:file` is on the *write* path too:
|
||||
;; `app.binfile.common/update-file!` derives its UPDATE columns from a file
|
||||
;; map's keys, so declaring `:backend` there made it try to write a `backend`
|
||||
;; column, which the `file` table does not have — it is synthesized on read.
|
||||
;; A projection `:extra` is local to the graph and cannot reach a write.
|
||||
;;
|
||||
;; `:options` is lifted out of `:data` before the blob is dropped
|
||||
;; (`app.graph.projection.document/document-attrs`); the rest come off the file
|
||||
;; map as `get-file` returns it.
|
||||
:extra [:map
|
||||
[:options {:optional true} [:maybe :map]]
|
||||
[:backend {:optional true} [:maybe :string]]
|
||||
[:comment-thread-seqn {:optional true} [:maybe :int]]
|
||||
[:ignore-sync-until {:optional true} [:maybe ::ct/inst]]]})
|
||||
|
||||
(def ^:private page-projection
|
||||
{:source ctp/schema:page
|
||||
:drop [:objects]})
|
||||
|
||||
(def ^:private component-projection
|
||||
{:source ctk/schema:component
|
||||
:drop [:objects]
|
||||
;; Soft-delete flag used at runtime; not in schema:component.
|
||||
:extra [:map
|
||||
[:deleted {:optional true} :boolean]
|
||||
[:annotation {:optional true} :string]]})
|
||||
|
||||
(def ^:private shape-projection
|
||||
{:drop [:type]})
|
||||
|
||||
(def ^:private shape-node-types
|
||||
[{:table "Frame" :penpot-type :frame :container? true}
|
||||
{:table "Group" :penpot-type :group :container? true}
|
||||
{:table "Boolean" :penpot-type :bool :container? true}
|
||||
{:table "SVGRaw" :penpot-type :svg-raw :container? true}
|
||||
{:table "Rectangle" :penpot-type :rect}
|
||||
{:table "Circle" :penpot-type :circle}
|
||||
{:table "Path" :penpot-type :path}
|
||||
{:table "Text" :penpot-type :text}
|
||||
{:table "Image" :penpot-type :image}])
|
||||
|
||||
(defn- resolve-schema
|
||||
[{:keys [schema source drop extra penpot-type]}]
|
||||
(or schema
|
||||
(when penpot-type
|
||||
(projection/project-shape-schema penpot-type
|
||||
{:drop drop
|
||||
:extra extra}))
|
||||
(projection/project-schema source
|
||||
{:drop drop
|
||||
:extra extra})))
|
||||
|
||||
(defn- shape-node-entry
|
||||
[{:keys [table penpot-type container?] :as entry}]
|
||||
(let [projection (-> shape-projection
|
||||
(merge (:projection entry))
|
||||
(assoc :penpot-type penpot-type))]
|
||||
{:table table
|
||||
:pk :id
|
||||
:penpot-type penpot-type
|
||||
:container? container?
|
||||
:projection projection
|
||||
:schema (resolve-schema projection)}))
|
||||
|
||||
(def node-types
|
||||
"Ordered node registry."
|
||||
(into [{:table "Document"
|
||||
:pk :id
|
||||
:projection document-projection
|
||||
:schema (resolve-schema document-projection)}
|
||||
{:table "Page"
|
||||
:pk :id
|
||||
:projection page-projection
|
||||
:schema (resolve-schema page-projection)}
|
||||
{:table "Component"
|
||||
:pk :id
|
||||
:projection component-projection
|
||||
:schema (resolve-schema component-projection)}]
|
||||
(map shape-node-entry shape-node-types)))
|
||||
|
||||
(def ^:private by-table
|
||||
(into {} (map (juxt :table identity) node-types)))
|
||||
|
||||
(def ^:private by-penpot-type
|
||||
(into {} (keep (fn [{:keys [penpot-type table]}]
|
||||
(when penpot-type [penpot-type table]))
|
||||
node-types)))
|
||||
|
||||
(def container-tables
|
||||
(into #{} (comp (filter :container?) (map :table)) node-types))
|
||||
|
||||
(def shape-tables
|
||||
(into [] (comp (filter :penpot-type) (map :table)) node-types))
|
||||
|
||||
(defn table-for-type
|
||||
"Map a Penpot shape `:type` keyword to a Ladybug node table name."
|
||||
[penpot-type]
|
||||
(get by-penpot-type (keyword penpot-type)))
|
||||
|
||||
(defn node-entry
|
||||
[table]
|
||||
(get by-table table))
|
||||
|
||||
(defn projection-for
|
||||
"Return the projection options map for `table`."
|
||||
[table]
|
||||
(:projection (node-entry table)))
|
||||
|
||||
(defn- entry-child-schema
|
||||
"Return the value schema from a Malli map entry (`[k s]` or `[k props s]`)."
|
||||
[entry]
|
||||
(if (> (count entry) 2)
|
||||
(nth entry 2)
|
||||
(nth entry 1)))
|
||||
|
||||
(defn column-name
|
||||
"Graph column name for projected key `k` on `table`."
|
||||
[_table k]
|
||||
(contract/column-name k))
|
||||
|
||||
(defn column-ladybug-type
|
||||
"Ladybug column type for projected key `k` on `table`."
|
||||
[table k]
|
||||
(some (fn [entry]
|
||||
(when (= k (first entry))
|
||||
(contract/ladybug-type (column-name table k)
|
||||
(types/ladybug-type (entry-child-schema entry)))))
|
||||
(projection/schema-map-entries (:schema (node-entry table)))))
|
||||
|
||||
(defn column-keys
|
||||
"Projected column keys for `table`, in registry order.
|
||||
|
||||
Keys the contract drops on this table are omitted, so the column order, the
|
||||
Arrow batch, and the DDL cannot disagree about what exists."
|
||||
[table]
|
||||
(into []
|
||||
(comp (map first)
|
||||
(remove #(contract/drop-key? table %)))
|
||||
(projection/schema-map-entries (:schema (node-entry table)))))
|
||||
|
||||
(defn columns
|
||||
"Projected column names for `table`, in registry order."
|
||||
[table]
|
||||
(mapv #(column-name table %) (column-keys table)))
|
||||
|
||||
(def ^:private validate-node-fn
|
||||
(memoize
|
||||
(fn [table]
|
||||
(let [{:keys [schema]} (node-entry table)]
|
||||
(sm/check-fn schema
|
||||
:type :validation
|
||||
:code (keyword "graph-node-projection" (str/lower-case table))
|
||||
:hint (str "invalid graph node projection for " table))))))
|
||||
|
||||
(defn- projection-error-hint
|
||||
[table explain]
|
||||
(str "invalid graph node projection for " table
|
||||
(when explain
|
||||
(str "\n" (sm/humanize-explain explain)))))
|
||||
|
||||
(defn validate-node
|
||||
"Validate and return projected node attrs for `table`."
|
||||
[table value]
|
||||
(let [{:keys [schema]} (node-entry table)]
|
||||
(try
|
||||
((validate-node-fn table) value)
|
||||
(catch clojure.lang.ExceptionInfo e
|
||||
(let [data (ex-data e)
|
||||
explain (or (::sm/explain data)
|
||||
(sm/explain schema value))]
|
||||
(ex/raise :type :validation
|
||||
:code (keyword "graph-node-projection" (str/lower-case table))
|
||||
:hint (projection-error-hint table explain)
|
||||
:table table
|
||||
::sm/explain explain
|
||||
:cause e))))))
|
||||
|
||||
(defn- get-projected-attr
|
||||
"The attribute under `k`, keyword or string key.
|
||||
|
||||
`if-some`, not `or`: `false` and `0` are values, and falling through on them
|
||||
is how `opacity 0` became `nil` and then the column default."
|
||||
[attrs k]
|
||||
(if-some [v (get attrs k)]
|
||||
v
|
||||
(when (keyword? k) (get attrs (name k)))))
|
||||
|
||||
(defn- raise-empty-projection!
|
||||
[table attrs]
|
||||
(ex/raise :type :validation
|
||||
:code (keyword "graph-node-projection" (str/lower-case table))
|
||||
:hint (str "empty graph node projection for " table
|
||||
"; columns=" (count (column-keys table))
|
||||
" shape-keys=" (vec (keys attrs)))))
|
||||
|
||||
(defn project-attrs
|
||||
"Select and validate the projected columns for `table` from `attrs`."
|
||||
[table attrs]
|
||||
;; `some?`, not truthiness: `false` and `0` are values. Dropping them sent
|
||||
;; `opacity 0` to the column default of 1.0 — a fully transparent shape
|
||||
;; projected as opaque.
|
||||
(let [projected (into {}
|
||||
(keep (fn [k]
|
||||
(let [v (get-projected-attr attrs k)]
|
||||
(when (some? v) [k v])))
|
||||
(column-keys table)))]
|
||||
(when (empty? projected)
|
||||
(raise-empty-projection! table attrs))
|
||||
(validate-node table projected)))
|
||||
|
||||
(defn match-label
|
||||
"Cypher node label for MATCH; backtick-wrapped when required by Ladybug."
|
||||
[table]
|
||||
(if (#{"Group" "Boolean"} table)
|
||||
(str "`" table "`")
|
||||
table))
|
||||
|
||||
(defn cypher-property-key
|
||||
"Backtick-wrapped column name for inline Cypher literals."
|
||||
[table k]
|
||||
(str "`" (column-name table k) "`"))
|
||||
|
||||
(defn column-map-key-fn
|
||||
"How a MAP column of `table` renders its keys.
|
||||
|
||||
A MAP's keys are values, not schema, so they keep the spelling their consumer
|
||||
parsed — `applied_tokens` is keyed in camelCase. Both writers need this, so it
|
||||
lives next to the column's type rather than in either of them."
|
||||
[table k]
|
||||
(contract/map-key-fn (column-name table k)))
|
||||
|
||||
(defn format-column-value
|
||||
"Cypher literal for `v` in column `k` of `table`.
|
||||
|
||||
The single place that knows both the column's Ladybug type and the contract
|
||||
detail that a MAP column may render its keys differently from `name` — used
|
||||
by the bulk loader's post-COPY fixups and by the incremental sync alike, so
|
||||
the two cannot disagree about a value's shape."
|
||||
[table k v]
|
||||
(ladybug/format-typed-value (column-ladybug-type table k)
|
||||
v
|
||||
(column-map-key-fn table k)))
|
||||
|
||||
(defn- create-node-table-ddl
|
||||
[{:keys [table pk]}]
|
||||
(let [cols (for [k (column-keys table)]
|
||||
(str "`" (column-name table k) "` " (column-ladybug-type table k)))]
|
||||
(str "CREATE NODE TABLE `" table "` ("
|
||||
(str/join ", " (concat cols
|
||||
[(str "PRIMARY KEY (`" (column-name table pk) "`)")]))
|
||||
");")))
|
||||
|
||||
(defn is-child-of-ddl
|
||||
[]
|
||||
(str "CREATE REL TABLE `IsChildOf` ("
|
||||
"FROM `Page` TO `Document`, "
|
||||
"FROM `Component` TO `Document`, "
|
||||
(str/join ", "
|
||||
(concat
|
||||
(map (fn [shape]
|
||||
(str "FROM `" shape "` TO `Page`"))
|
||||
shape-tables)
|
||||
(for [shape shape-tables
|
||||
container container-tables]
|
||||
(str "FROM `" shape "` TO `" container "`"))))
|
||||
", `position` INT64);"))
|
||||
|
||||
(defn is-instance-of-ddl
|
||||
"Frame instance heads → Component."
|
||||
[]
|
||||
"CREATE REL TABLE `IsInstanceOf` (FROM `Frame` TO `Component`);")
|
||||
|
||||
(defn- shape-to-shape-rel-ddl
|
||||
"A rel table over the full shape × shape product.
|
||||
|
||||
Created up-front rather than on demand: the bulk loader must never race on
|
||||
lazy table creation, and a consumer can then tell \"this producer cannot
|
||||
emit that pair\" from \"this document happens to have none\"."
|
||||
[rel props]
|
||||
(str "CREATE REL TABLE `" rel "` ("
|
||||
(str/join ", " (for [from shape-tables
|
||||
to shape-tables]
|
||||
(str "FROM `" from "` TO `" to "`")))
|
||||
(when (seq props) (str ", " (str/join ", " props)))
|
||||
");"))
|
||||
|
||||
(defn refers-to-ddl
|
||||
"Instance shape → its homologue in the component main instance, resolved
|
||||
from `shape-ref`."
|
||||
[]
|
||||
(shape-to-shape-rel-ddl "RefersTo" nil))
|
||||
|
||||
(defn fills-swap-slot-ddl
|
||||
"Swapped-in shape → the slot shape it replaces."
|
||||
[]
|
||||
(shape-to-shape-rel-ddl "FillsSwapSlot" ["`slot_id` UUID"]))
|
||||
|
||||
(defn ddl-statements
|
||||
[]
|
||||
(-> (mapv create-node-table-ddl node-types)
|
||||
(conj (is-child-of-ddl))
|
||||
(conj (is-instance-of-ddl))
|
||||
(conj (refers-to-ddl))
|
||||
(conj (fills-swap-slot-ddl))))
|
||||
@@ -0,0 +1,85 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema.projection
|
||||
"Derive Ladybug node column schemas from Penpot Malli sources.
|
||||
|
||||
Start from the canonical schema and remove the keys that must not become
|
||||
graph columns."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.types.shape :as cts]
|
||||
[malli.core :as m]))
|
||||
|
||||
(def ^:private malli-opts sm/default-options)
|
||||
|
||||
(defn- coerce-schema
|
||||
"Normalize Malli sources to a compiled schema, unwrapping `:val` nodes."
|
||||
[schema]
|
||||
(loop [s (cond
|
||||
(sm/schema? schema) schema
|
||||
:else (sm/schema schema))]
|
||||
(if (= :malli.core/val (sm/type s))
|
||||
(recur (first (sm/children s)))
|
||||
s)))
|
||||
|
||||
(defn- unsupported-projection-schema!
|
||||
[schema]
|
||||
(ex/raise :type :internal
|
||||
:code :unsupported-projection-schema
|
||||
:hint (str "unsupported projection schema type: "
|
||||
(sm/type (coerce-schema schema)))))
|
||||
|
||||
(defn schema-map-entries
|
||||
"Map entries for `schema`, flattening `:merge` composites."
|
||||
[schema]
|
||||
(let [s (coerce-schema schema)]
|
||||
(or (seq (sm/entries s))
|
||||
(unsupported-projection-schema! schema))))
|
||||
|
||||
(defn- select-projected-keys
|
||||
"Project `schema` to a flat map schema, optionally dropping keys."
|
||||
[schema drop-keys]
|
||||
(let [s (coerce-schema schema)
|
||||
keys (if (seq drop-keys)
|
||||
(remove (set drop-keys) (sm/keys s))
|
||||
(sm/keys s))]
|
||||
(sm/select-keys s (vec keys))))
|
||||
|
||||
(defn shape-type-schema
|
||||
"Return the compiled Penpot Malli branch for shape type `penpot-type`.
|
||||
|
||||
`m/entries` on the shape `:multi` yields MapEntries whose values are
|
||||
compiled branch schemas (wrapped in `:val`). `m/children` returns raw
|
||||
entry forms and must not be used here."
|
||||
[penpot-type]
|
||||
(let [kw (keyword penpot-type)
|
||||
multi (sm/schema cts/schema:shape-attrs)]
|
||||
(or (some (fn [entry]
|
||||
(when (= kw (key entry))
|
||||
(val entry)))
|
||||
(m/entries multi malli-opts))
|
||||
(ex/raise :type :validation
|
||||
:code :unknown-shape-type
|
||||
:hint (str "unknown penpot shape type: " kw)))))
|
||||
|
||||
(defn project-schema
|
||||
"Build a graph node schema from canonical Malli `source`.
|
||||
|
||||
Options:
|
||||
- `:drop` - keys removed from the source
|
||||
- `:extra` - optional extra `[:map ...]` merged on top"
|
||||
[source {:keys [drop extra]}]
|
||||
(let [projected (select-projected-keys source drop)]
|
||||
(if extra
|
||||
(sm/merge projected (coerce-schema extra))
|
||||
projected)))
|
||||
|
||||
(defn project-shape-schema
|
||||
"Project `:drop` from the Penpot schema for `penpot-type`."
|
||||
[penpot-type opts]
|
||||
(project-schema (shape-type-schema penpot-type) opts))
|
||||
@@ -0,0 +1,174 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema.types
|
||||
"Map Malli schemas to Ladybug column types.
|
||||
|
||||
Ladybug is schema-first and strongly typed: every property key gets its type
|
||||
at table-creation time, and there is no widening later. That makes this
|
||||
mapping the whole of the graph's typing, and it is worth being tight — a
|
||||
column typed `DOUBLE[4]` is four numbers a consumer reads as a tensor row,
|
||||
where the same value as `JSON` is text somebody has to parse and trust. So
|
||||
JSON is the fallback of last resort, taken only where the Malli schema
|
||||
genuinely admits shapes no single column can hold.
|
||||
|
||||
Three groups, in the order the mapping tries them:
|
||||
|
||||
1. **Scalars** (`base-type->ladybug`) — the leaf Malli types.
|
||||
2. **Registered composites** (`custom-type->ladybug`) — Penpot's own value
|
||||
types whose *layout* is fixed even though Malli only sees a map or a
|
||||
string: a matrix is six doubles, a point two, a rect four, a hex colour
|
||||
one packed integer. These are named explicitly because the tight encoding
|
||||
is a modelling decision, not something derivable from the schema.
|
||||
3. **Structure** — collections become `T[]`, `:map-of` becomes `MAP(k, v)`,
|
||||
and a closed map of scalars becomes a `STRUCT`. Anything that could be
|
||||
more than one shape (a `:multi`, an `:or`, an optional-keyed map) becomes
|
||||
`JSON`, because a Ladybug column cannot be two types.
|
||||
|
||||
Every encoding here has a matching value formatter in `app.graph.ladybug`.
|
||||
The two must move together: a column type with no case there falls back to
|
||||
guessing the literal from the runtime value."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[clojure.string :as str]
|
||||
[malli.core :as m]))
|
||||
|
||||
(def ^:private malli-opts sm/default-options)
|
||||
|
||||
(def ^:private base-type->ladybug
|
||||
{::sm/uuid "UUID"
|
||||
::sm/safe-number "DOUBLE"
|
||||
::sm/safe-double "DOUBLE"
|
||||
::sm/safe-int "INT64"
|
||||
::sm/number "DOUBLE"
|
||||
::sm/boolean "BOOLEAN"
|
||||
::sm/int "INT64"
|
||||
::ct/inst "TIMESTAMP"
|
||||
:uuid "UUID"
|
||||
:string "STRING"
|
||||
:int "INT64"
|
||||
:double "DOUBLE"
|
||||
:float "DOUBLE"
|
||||
:boolean "BOOLEAN"
|
||||
:keyword "STRING"
|
||||
:inst "TIMESTAMP"})
|
||||
|
||||
(def ^:private custom-type->ladybug
|
||||
"Penpot value types with a fixed layout Malli does not express.
|
||||
|
||||
Fixed-size arrays are the point of each: they are dense, they need no
|
||||
parsing, and a consumer can read a whole column as a tensor.
|
||||
|
||||
- `::gmt/matrix` — the affine transform, `[a b c d e f]`.
|
||||
- `::gpt/point` — `[x y]`.
|
||||
- `::grc/rect` — `[x y width height]`. `x1`/`y1`/`x2`/`y2` are dropped: they
|
||||
are derivable from those four, and carrying them would double the column.
|
||||
- `::clr/hex-color` — `#RRGGBB` packed as `0xRRGGBBAA`, so colours compare
|
||||
and group without string handling."
|
||||
{:app.common.geom.matrix/matrix "DOUBLE[6]"
|
||||
:app.common.geom.point/point "DOUBLE[2]"
|
||||
:app.common.geom.rect/rect "DOUBLE[4]"
|
||||
:app.common.types.color/hex-color "UINT32"})
|
||||
|
||||
(def ^:private collection-types
|
||||
#{:vector :sequential :set ::sm/vec ::sm/set ::sm/coll})
|
||||
|
||||
(def ^:private string-collection-types
|
||||
"Registered collection schemas whose element type is not in `children`."
|
||||
{::sm/set-of-strings "STRING[]"
|
||||
::sm/set-of-keywords "STRING[]"
|
||||
::sm/set-of-uuid "UUID[]"
|
||||
::sm/vec-of-uuid "UUID[]"})
|
||||
|
||||
(defn- normalize-schema
|
||||
"Resolve refs, but stop at a schema this namespace maps explicitly.
|
||||
|
||||
Order matters: `::grc/rect` derefs to an `:and` over a map, and following
|
||||
that would lose the fixed-size-array encoding."
|
||||
[schema]
|
||||
(let [s (sm/schema schema)]
|
||||
(if (and (m/-ref-schema? s)
|
||||
(not (contains? custom-type->ladybug (m/type s)))
|
||||
(not (contains? string-collection-types (m/type s))))
|
||||
(recur (m/deref s malli-opts))
|
||||
s)))
|
||||
|
||||
(declare ladybug-type)
|
||||
|
||||
(defn- entry-child
|
||||
"The value schema of a Malli map entry (`[k s]` or `[k props s]`)."
|
||||
[entry]
|
||||
(if (> (count entry) 2) (nth entry 2) (nth entry 1)))
|
||||
|
||||
(defn- entry-optional?
|
||||
[entry]
|
||||
(and (> (count entry) 2)
|
||||
(:optional (nth entry 1))))
|
||||
|
||||
(defn- struct-type
|
||||
"`STRUCT(...)` for a closed map of scalars, or nil when JSON is the honest answer.
|
||||
|
||||
A struct is a fixed layout: every field present, every field a single type.
|
||||
An optional key would make the column's shape depend on the row, and a nested
|
||||
collection or map makes it recursive — Ladybug allows nesting, but a consumer
|
||||
reading such a column gains nothing over JSON, so the line is drawn at
|
||||
scalars."
|
||||
[s]
|
||||
(let [entries (m/entries s malli-opts)]
|
||||
(when (and (seq entries)
|
||||
(not-any? entry-optional? entries))
|
||||
(let [fields (for [entry entries
|
||||
:let [t (ladybug-type (entry-child entry))]]
|
||||
(when (and t
|
||||
(not= "JSON" t)
|
||||
(not (str/includes? t "(")))
|
||||
;; snake_case like a column name, and always
|
||||
;; backtick-quoted: a grid cell has a field called
|
||||
;; `column`, which is a Ladybug keyword, and an unquoted
|
||||
;; one fails to parse in the DDL *and* in every literal.
|
||||
;; The catalog reports them unquoted.
|
||||
(str "`" (str/replace (name (key entry)) "-" "_") "` " t)))]
|
||||
(when (every? some? fields)
|
||||
(str "STRUCT(" (str/join ", " fields) ")"))))))
|
||||
|
||||
(defn ladybug-type
|
||||
"Return the Ladybug column type for a Malli child schema."
|
||||
[schema]
|
||||
(let [s (normalize-schema schema)
|
||||
t (m/type s)]
|
||||
(or (base-type->ladybug t)
|
||||
(custom-type->ladybug t)
|
||||
(string-collection-types t)
|
||||
(when (contains? collection-types t)
|
||||
(when-let [child (first (m/children s malli-opts))]
|
||||
(str (ladybug-type child) "[]")))
|
||||
(case t
|
||||
(:maybe :and) (ladybug-type (first (m/children s malli-opts)))
|
||||
|
||||
;; `::sm/one-of` is how Penpot spells a closed set of keywords —
|
||||
;; `:blend-mode`, `:grow-type`, every `:layout-*`. One keyword, one
|
||||
;; string.
|
||||
(:enum ::sm/one-of) "STRING"
|
||||
|
||||
:map-of
|
||||
(let [[key-schema value-schema] (m/children s malli-opts)]
|
||||
(str "MAP(" (ladybug-type key-schema) ", "
|
||||
(ladybug-type value-schema) ")"))
|
||||
|
||||
:map (or (struct-type s) "JSON")
|
||||
|
||||
;; A schema we do not recognize. If it has no children it is a leaf —
|
||||
;; one of Penpot's registered keyword or enum schemas, say — and a
|
||||
;; string holds it exactly. If it has children it is a composite whose
|
||||
;; shape we cannot pin down, and JSON is the honest answer.
|
||||
(if (empty? (m/children s malli-opts))
|
||||
"STRING"
|
||||
(do
|
||||
(l/wrn :hint "unmapped composite malli type, defaulting to JSON"
|
||||
:malli-type t)
|
||||
"JSON"))))))
|
||||
@@ -0,0 +1,202 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema.values
|
||||
"Shape a Penpot value into the plain data its Ladybug column type wants.
|
||||
|
||||
Ladybug is strongly typed, and `app.graph.schema.types` maps Penpot's Malli
|
||||
schemas onto types as tight as it can — a matrix is `DOUBLE[6]`, a rect
|
||||
`DOUBLE[4]`, a colour `UINT32`, a closed map a `STRUCT`. A tight column is
|
||||
only worth having if the writer actually fills it in that shape, which is
|
||||
what this namespace does: it turns records and maps into the numbers, vectors
|
||||
and plain maps the type names.
|
||||
|
||||
It deliberately stops there. Serialization belongs to the writer — Cypher
|
||||
literals in `app.graph.ladybug`, Arrow vectors in `app.graph.arrow` — so that
|
||||
shaping a value and writing it are separate concerns and each has one home.
|
||||
|
||||
The type language is the Ladybug one, read recursively: `T[]`, `T[n]`,
|
||||
`MAP(k, v)`, `STRUCT(name t, …)`. Anything else is passed through."
|
||||
(:require
|
||||
[app.common.geom.matrix :as gmt]
|
||||
[app.common.geom.point :as gpt]
|
||||
[app.common.types.color :as clr]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn- split-args
|
||||
"Split a comma-separated type argument list, respecting nesting.
|
||||
|
||||
`\"UUID, STRUCT(a INT64, b INT64)\"` → `[\"UUID\" \"STRUCT(a INT64, b INT64)\"]`."
|
||||
[s]
|
||||
(loop [chars (seq s) depth 0 current (StringBuilder.) out []]
|
||||
(if-let [c (first chars)]
|
||||
(cond
|
||||
(and (= c \,) (zero? depth))
|
||||
(recur (rest chars) depth (StringBuilder.) (conj out (str/trim (str current))))
|
||||
|
||||
(or (= c \() (= c \[))
|
||||
(recur (rest chars) (inc depth) (.append current c) out)
|
||||
|
||||
(or (= c \)) (= c \]))
|
||||
(recur (rest chars) (dec depth) (.append current c) out)
|
||||
|
||||
:else
|
||||
(recur (rest chars) depth (.append current c) out))
|
||||
(let [last-arg (str/trim (str current))]
|
||||
(cond-> out (seq last-arg) (conj last-arg))))))
|
||||
|
||||
(defn- parse-list
|
||||
"`[element-type]` when `t` is a list or fixed-size array type, else nil.
|
||||
|
||||
`DOUBLE[]` and `DOUBLE[4]` are both lists of doubles as far as shaping goes;
|
||||
the size only matters to the DDL."
|
||||
[t]
|
||||
(when-let [[_ element] (re-matches #"(.+?)\[\d*\]$" t)]
|
||||
[element]))
|
||||
|
||||
(defn- parse-map
|
||||
"`[key-type value-type]` when `t` is a MAP type, else nil."
|
||||
[t]
|
||||
(when-let [[_ args] (re-matches #"MAP\((.*)\)$" t)]
|
||||
(let [[k v] (split-args args)]
|
||||
(when (and k v) [k v]))))
|
||||
|
||||
(defn- parse-struct
|
||||
"`[[field-name field-type] …]` when `t` is a STRUCT type, else nil.
|
||||
|
||||
Field names arrive backtick-quoted (see `app.graph.schema.types`). The
|
||||
quoting is syntax, so it is stripped by default and re-applied by the writer —
|
||||
except for the Arrow writer, which needs it kept (`keep-quotes?`)."
|
||||
[t keep-quotes?]
|
||||
(when-let [[_ args] (re-matches #"STRUCT\((.*)\)$" t)]
|
||||
(for [arg (split-args args)
|
||||
:let [idx (str/index-of arg " ")]
|
||||
:when idx]
|
||||
[(cond-> (subs arg 0 idx) (not keep-quotes?) (str/replace "`" ""))
|
||||
(str/trim (subs arg (inc idx)))])))
|
||||
|
||||
(def ^:private struct-field-keys
|
||||
"Field name → the Penpot keys that may hold it.
|
||||
|
||||
A STRUCT field name is the snake_case of the Penpot key, but a value arrives
|
||||
with its original key, and some arrive from JSON with the string form. Both
|
||||
are tried before giving up."
|
||||
(memoize
|
||||
(fn [field]
|
||||
[(keyword (str/replace field "_" "-"))
|
||||
(keyword field)
|
||||
field
|
||||
(str/replace field "_" "-")])))
|
||||
|
||||
(defn- struct-field
|
||||
[value field]
|
||||
(some (fn [k] (when (contains? value k) (get value k)))
|
||||
(struct-field-keys field)))
|
||||
|
||||
(defn- fixed-vector
|
||||
"`v` as a plain vector of numbers, for a `DOUBLE[n]` column.
|
||||
|
||||
Records come first because they are what a realized snapshot holds; the map
|
||||
forms are what a JSON round-trip leaves behind."
|
||||
[v]
|
||||
(cond
|
||||
(gmt/matrix? v) [(:a v) (:b v) (:c v) (:d v) (:e v) (:f v)]
|
||||
(gpt/point? v) [(:x v) (:y v)]
|
||||
|
||||
;; A rect: four of the eight fields, the rest being derivable.
|
||||
(and (map? v) (contains? v :width) (contains? v :height))
|
||||
[(:x v) (:y v) (:width v) (:height v)]
|
||||
|
||||
(and (map? v) (contains? v :x) (contains? v :y))
|
||||
[(:x v) (:y v)]
|
||||
|
||||
(and (map? v) (contains? v :a) (contains? v :f))
|
||||
[(:a v) (:b v) (:c v) (:d v) (:e v) (:f v)]
|
||||
|
||||
(sequential? v) (vec v)
|
||||
:else nil))
|
||||
|
||||
(defn- packed-color
|
||||
"`#RRGGBB` as the packed integer `0xRRGGBBAA`.
|
||||
|
||||
Alpha defaults to opaque: the column holds a colour, and any opacity Penpot
|
||||
keeps alongside it is a separate attribute."
|
||||
[v]
|
||||
(cond
|
||||
(integer? v) v
|
||||
(and (string? v) (clr/valid-hex-color? v))
|
||||
(let [rgb (Long/parseLong (subs v 1) 16)]
|
||||
(bit-or (bit-shift-left rgb 8) 0xFF))
|
||||
:else nil))
|
||||
|
||||
(def struct-fields
|
||||
"`[[field-name field-type] …]` for a STRUCT type, memoized.
|
||||
|
||||
Public because the writers need the same field list to emit a literal."
|
||||
(memoize (fn [ladybug-type] (vec (parse-struct ladybug-type false)))))
|
||||
|
||||
(def struct-fields-quoted
|
||||
"`struct-fields` with the DDL's backticks intact.
|
||||
|
||||
Only the Arrow writer wants this: Ladybug names a staged struct's fields from
|
||||
the Arrow child names and quotes none of them, so a field whose name is a
|
||||
reserved word — a layout grid cell's `column` — has to arrive already quoted
|
||||
or `createArrowTable` fails outright."
|
||||
(memoize (fn [ladybug-type] (vec (parse-struct ladybug-type true)))))
|
||||
|
||||
(def map-types
|
||||
"`[key-type value-type]` for a MAP type, memoized."
|
||||
(memoize (fn [ladybug-type] (parse-map ladybug-type))))
|
||||
|
||||
(def list-element
|
||||
"Element type of a `T[]` / `T[n]` column, memoized; nil when not a list."
|
||||
(memoize (fn [ladybug-type] (first (parse-list ladybug-type)))))
|
||||
|
||||
(declare coerce)
|
||||
|
||||
(defn- coerce-struct
|
||||
[fields v]
|
||||
(when (map? v)
|
||||
(into {}
|
||||
(keep (fn [[field field-type]]
|
||||
(when-some [fv (struct-field v field)]
|
||||
[field (coerce field-type fv)])))
|
||||
fields)))
|
||||
|
||||
(defn coerce
|
||||
"`v` as the plain data a column of `ladybug-type` holds.
|
||||
|
||||
Returns `nil` when the value cannot be shaped that way, which callers treat
|
||||
as \"write NULL\" — a wrong shape in a strongly typed column fails the whole
|
||||
load, so declining is better than guessing."
|
||||
[ladybug-type v]
|
||||
(cond
|
||||
(nil? v) nil
|
||||
(not (string? ladybug-type)) v
|
||||
|
||||
(= "UINT32" ladybug-type) (packed-color v)
|
||||
|
||||
;; Fixed-size numeric arrays are records: matrix, point, rect.
|
||||
(re-matches #"DOUBLE\[\d+\]" ladybug-type) (fixed-vector v)
|
||||
|
||||
:else
|
||||
(if-let [[element] (parse-list ladybug-type)]
|
||||
(when (or (sequential? v) (set? v))
|
||||
;; A set has no order, so its column would otherwise vary between
|
||||
;; builds of the same file. Sorting makes it deterministic — which is
|
||||
;; what lets two builds be diffed at all, and what a stable golden
|
||||
;; needs. Sequential values keep their order: for `shapes` and
|
||||
;; `points`, the order *is* the content.
|
||||
(let [elements (mapv #(coerce element %) v)]
|
||||
(if (set? v) (vec (sort-by str elements)) elements)))
|
||||
(if-let [[key-type value-type] (parse-map ladybug-type)]
|
||||
(when (map? v)
|
||||
(into {}
|
||||
(map (fn [[k mv]] [(coerce key-type k) (coerce value-type mv)]))
|
||||
v))
|
||||
(if-let [fields (seq (parse-struct ladybug-type false))]
|
||||
(coerce-struct fields v)
|
||||
v)))))
|
||||
@@ -0,0 +1,48 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.stats
|
||||
(:require
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes]))
|
||||
|
||||
(defn- count-on-connection
|
||||
[conn statement]
|
||||
(or (ladybug/query-scalar-on-connection! conn statement) 0))
|
||||
|
||||
(defn- rel-table-names
|
||||
"Relationship tables present in the open database.
|
||||
|
||||
Read from the catalog so a newly ported transform's edges are counted
|
||||
without this namespace being told about it."
|
||||
[conn]
|
||||
(->> (ladybug/query-on-connection!
|
||||
conn "CALL show_tables() WHERE type = 'REL' RETURN name;" :max-rows 1000)
|
||||
:rows
|
||||
(map first)))
|
||||
|
||||
(defn summarize-connection
|
||||
"Return node/edge counts using an open Ladybug connection."
|
||||
[conn]
|
||||
{:nodes (into {}
|
||||
(map (fn [table]
|
||||
[table (count-on-connection
|
||||
conn
|
||||
(str "MATCH (n:" (nodes/match-label table) ") "
|
||||
"RETURN count(n) AS " table "_c;"))])
|
||||
(map :table nodes/node-types)))
|
||||
:edges (into {}
|
||||
(map (fn [rel]
|
||||
[(keyword rel)
|
||||
(count-on-connection
|
||||
conn
|
||||
(str "MATCH ()-[e:`" rel "`]->() RETURN count(e) AS c;"))]))
|
||||
(rel-table-names conn))})
|
||||
|
||||
(defn summarize
|
||||
"Return node/edge counts from the graph database."
|
||||
[db-path]
|
||||
(ladybug/with-connection! db-path summarize-connection))
|
||||
@@ -0,0 +1,900 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.sync
|
||||
"Incremental Ladybug graph updates from Penpot file-change events."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.projection.document :as projection.document]
|
||||
[app.graph.schema.nodes :as nodes]
|
||||
[clojure.string :as str])
|
||||
(:import
|
||||
com.ladybugdb.Connection))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:private supported-change-types
|
||||
#{:add-obj :mod-obj :del-obj
|
||||
:add-page :del-page :mod-page :mov-objects
|
||||
:add-component :mod-component :del-component
|
||||
:restore-component :purge-component})
|
||||
|
||||
(defn- shape-table
|
||||
[shape]
|
||||
(nodes/table-for-type (:type shape)))
|
||||
|
||||
(defn- build-parent-map
|
||||
[edges]
|
||||
(into {}
|
||||
(map (fn [{:keys [from-id to-id to-table]}]
|
||||
[from-id {:parent-id to-id :parent-table to-table}]))
|
||||
edges))
|
||||
|
||||
(defn- build-children-map
|
||||
[edges]
|
||||
(reduce (fn [acc {:keys [from-id to-id]}]
|
||||
(update acc to-id (fnil conj #{}) from-id))
|
||||
{}
|
||||
edges))
|
||||
|
||||
(defn- resolve-page-id
|
||||
[shape-id parents pages]
|
||||
(loop [id shape-id]
|
||||
(cond
|
||||
(contains? pages id) id
|
||||
(get parents id) (recur (:parent-id (parents id)))
|
||||
:else nil)))
|
||||
|
||||
(defn- node-attrs-id
|
||||
[attrs]
|
||||
(cond
|
||||
(map? attrs) (or (:id attrs) (get attrs "id"))
|
||||
(and (vector? attrs) (= 2 (count attrs)))
|
||||
(let [[k v] attrs]
|
||||
(when (or (= k :id) (= k "id")) v))))
|
||||
|
||||
(defn- table-rows
|
||||
"Normalize a projection table value to a vector of attribute maps."
|
||||
[nodes table]
|
||||
(let [rows (or (get nodes table) (get nodes (keyword table)))]
|
||||
(cond
|
||||
(nil? rows) []
|
||||
(map? rows) [rows]
|
||||
(sequential? rows) (vec rows)
|
||||
:else [])))
|
||||
|
||||
(defn- document-id-from-nodes
|
||||
[nodes file-id]
|
||||
(or (some node-attrs-id (table-rows nodes "Document"))
|
||||
file-id))
|
||||
|
||||
(defn- page-index-entry
|
||||
[attrs]
|
||||
(let [id (node-attrs-id attrs)]
|
||||
[id {:id id
|
||||
:name (:name attrs)
|
||||
:index (long (:index attrs 0))}]))
|
||||
|
||||
(defn- index-pages
|
||||
[nodes]
|
||||
(into {} (map page-index-entry (table-rows nodes "Page"))))
|
||||
|
||||
(defn- component-index-entry
|
||||
[attrs]
|
||||
(let [id (node-attrs-id attrs)]
|
||||
[id {:id id
|
||||
:name (:name attrs)
|
||||
:deleted (boolean (:deleted attrs))}]))
|
||||
|
||||
(defn- index-components
|
||||
[nodes]
|
||||
(into {} (map component-index-entry (table-rows nodes "Component"))))
|
||||
|
||||
(defn- shape-index-table?
|
||||
[table]
|
||||
(not (contains? #{"Document" "Page" "Component"
|
||||
:Document :Page :Component}
|
||||
table)))
|
||||
|
||||
(defn- shape-index-entry
|
||||
[table attrs parents pages edges]
|
||||
(let [shape-id (node-attrs-id attrs)
|
||||
{:keys [parent-id parent-table]} (parents shape-id)
|
||||
edge (first (filter #(= shape-id (:from-id %)) edges))]
|
||||
[shape-id {:id shape-id
|
||||
:name (:name attrs)
|
||||
:table table
|
||||
:parent-id parent-id
|
||||
:parent-table parent-table
|
||||
:position (long (:position edge 0))
|
||||
:frame-id (:frame-id attrs)
|
||||
;; The projection already denormalized these; re-deriving
|
||||
;; page-id from the parent chain would only be a second way to
|
||||
;; get the same answer. `:component-ctx` is what later
|
||||
;; `:add-obj` children inherit — it is the shape's effective
|
||||
;; component-id, which loses the barrier case of a *non-Frame*
|
||||
;; carrying its own `component-id` (indistinguishable once
|
||||
;; denormalized). Cold projection keeps the distinction. Only a
|
||||
;; graph synced across such a shape can drift, and a Reload
|
||||
;; rebuilds it.
|
||||
:component-ctx (:component-id attrs)
|
||||
:page-id (or (:page-id attrs)
|
||||
(resolve-page-id shape-id parents pages))}]))
|
||||
|
||||
(defn- index-shapes
|
||||
[nodes edges parents pages]
|
||||
(reduce
|
||||
(fn [acc [table _]]
|
||||
(into acc (map #(shape-index-entry table % parents pages edges)
|
||||
(table-rows nodes table))))
|
||||
{}
|
||||
(filter (fn [[table _]] (shape-index-table? table)) nodes)))
|
||||
|
||||
(defn build-index
|
||||
"Build a sync index from a full graph projection."
|
||||
[file-id revn {:keys [nodes edges]}]
|
||||
(let [doc-id (document-id-from-nodes nodes file-id)
|
||||
pages (index-pages nodes)
|
||||
components (index-components nodes)
|
||||
parents (build-parent-map edges)
|
||||
children-index (build-children-map edges)
|
||||
shapes (index-shapes nodes edges parents pages)]
|
||||
{:file-id file-id
|
||||
:doc-id doc-id
|
||||
:revn (long revn)
|
||||
:pages pages
|
||||
:components components
|
||||
:shapes shapes
|
||||
:children children-index}))
|
||||
|
||||
|
||||
(defn- format-node-value
|
||||
[table k v]
|
||||
(nodes/format-column-value table k v))
|
||||
|
||||
(defn- create-node-statement
|
||||
[table attrs]
|
||||
(let [label (nodes/match-label table)
|
||||
pairs (for [k (nodes/column-keys table)
|
||||
:let [v (get attrs k)]
|
||||
:when (some? v)]
|
||||
(str (nodes/cypher-property-key table k) ": "
|
||||
(format-node-value table k v)))]
|
||||
(str "CREATE (:" label " {" (str/join ", " pairs) "});")))
|
||||
|
||||
(defn- delete-node-statement
|
||||
[table shape-id]
|
||||
(str "MATCH (n:" (nodes/match-label table) " {id: " (ladybug/format-uuid shape-id) "}) "
|
||||
"DETACH DELETE n;"))
|
||||
|
||||
(defn- create-edge-statement
|
||||
[{:keys [from-table from-id to-table to-id position]}]
|
||||
(str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "}), "
|
||||
"(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
|
||||
"CREATE (s)-[:IsChildOf {position: " (ladybug/format-int position) "}]->(p);"))
|
||||
|
||||
(defn- create-instance-of-statement
|
||||
"Link a Frame instance head to its Component.
|
||||
|
||||
No-op when the Component is absent (e.g. library component not ingested)."
|
||||
[frame-id component-id]
|
||||
(str "MATCH (f:Frame {id: " (ladybug/format-uuid frame-id) "}), "
|
||||
"(c:Component {id: " (ladybug/format-uuid component-id) "}) "
|
||||
"WHERE NOT COALESCE(c.deleted, false) "
|
||||
"MERGE (f)-[:IsInstanceOf]->(c);"))
|
||||
|
||||
(defn- delete-instance-of-statement
|
||||
[frame-id]
|
||||
(str "MATCH (f:Frame {id: " (ladybug/format-uuid frame-id) "})"
|
||||
"-[r:IsInstanceOf]->(:Component) "
|
||||
"DELETE r;"))
|
||||
|
||||
(defn- instance-of-statements
|
||||
"Cypher to (re)link `IsInstanceOf` after add/mod of a Frame's component-id."
|
||||
[table shape-id component-id]
|
||||
(when (= table "Frame")
|
||||
(cond-> [(delete-instance-of-statement shape-id)]
|
||||
(some? component-id)
|
||||
(conj (create-instance-of-statement shape-id component-id)))))
|
||||
|
||||
(defn- delete-edge-statement
|
||||
[{:keys [from-table from-id to-table to-id]}]
|
||||
(str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "})"
|
||||
"-[r:IsChildOf]->"
|
||||
"(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
|
||||
"DELETE r;"))
|
||||
|
||||
(defn- set-edge-position-statement
|
||||
[{:keys [from-table from-id to-table to-id position]}]
|
||||
(str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "})"
|
||||
"-[r:IsChildOf]->"
|
||||
"(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
|
||||
"SET r.position = " (ladybug/format-int position) ";"))
|
||||
|
||||
(defn- set-node-attr-statement
|
||||
[table shape-id attr value]
|
||||
(str "MATCH (s:" (nodes/match-label table) " {id: " (ladybug/format-uuid shape-id) "}) "
|
||||
"SET s." (nodes/cypher-property-key table attr) " = "
|
||||
(format-node-value table attr value) ";"))
|
||||
|
||||
(defn- set-page-name-statement
|
||||
[page-id name]
|
||||
(str "MATCH (p:Page {id: " (ladybug/format-uuid page-id) "}) "
|
||||
"SET p.name = " (ladybug/format-string name) ";"))
|
||||
|
||||
(defn- remove-node-attr-statement
|
||||
"Clear a property. Ladybug has no Neo4j-style REMOVE; SET to NULL."
|
||||
[table shape-id attr]
|
||||
(str "MATCH (s:" (nodes/match-label table) " {id: " (ladybug/format-uuid shape-id) "}) "
|
||||
"SET s." (nodes/cypher-property-key table attr) " = NULL;"))
|
||||
|
||||
(defn- index-add-component!
|
||||
[index {:keys [id name doc-id]}]
|
||||
(-> index
|
||||
(assoc-in [:components id] {:id id :name name :deleted false})
|
||||
(update :children update doc-id (fnil conj #{}) id)))
|
||||
|
||||
(defn- index-remove-component!
|
||||
[index component-id]
|
||||
(let [doc-id (:doc-id index)]
|
||||
(-> index
|
||||
(update :components dissoc component-id)
|
||||
(update :children update doc-id #(disj (or % #{}) component-id)))))
|
||||
|
||||
|
||||
(defn- set-document-revision-statement
|
||||
"Set the Document's revision number.
|
||||
|
||||
`app.graph.schema.contract` names the column `revision`, not `revn`. The name
|
||||
is produced by `nodes/cypher-property-key`, so this statement and the DDL
|
||||
cannot disagree."
|
||||
[doc-id revn]
|
||||
(str "MATCH (d:Document {id: " (ladybug/format-uuid doc-id) "}) "
|
||||
"SET d." (nodes/cypher-property-key "Document" :revn) " = "
|
||||
(ladybug/format-int revn) ";"))
|
||||
|
||||
(defn- resolve-parent-for-add
|
||||
[index {:keys [parent-id frame-id page-id]}]
|
||||
(let [pid (or parent-id frame-id)]
|
||||
(if (or (nil? pid) (uuid/zero? pid))
|
||||
(when page-id
|
||||
{:parent-id page-id :parent-table "Page"})
|
||||
(if-let [shape (get-in index [:shapes pid])]
|
||||
{:parent-id pid :parent-table (:table shape)}
|
||||
(when (get-in index [:pages pid])
|
||||
{:parent-id pid :parent-table "Page"})))))
|
||||
|
||||
(defn- index-add-shape!
|
||||
[index {:keys [id name table parent-id parent-table position page-id
|
||||
frame-id component-ctx]}]
|
||||
(-> index
|
||||
(assoc-in [:shapes id]
|
||||
{:id id
|
||||
:name name
|
||||
:table table
|
||||
:parent-id parent-id
|
||||
:parent-table parent-table
|
||||
:position position
|
||||
:frame-id frame-id
|
||||
:component-ctx component-ctx
|
||||
:page-id page-id})
|
||||
(update :children update parent-id (fnil conj #{}) id)))
|
||||
|
||||
(defn- index-remove-shape!
|
||||
[index shape-id]
|
||||
(if-let [shape (get-in index [:shapes shape-id])]
|
||||
(-> index
|
||||
(update :shapes dissoc shape-id)
|
||||
(update :children update (:parent-id shape)
|
||||
#(disj (or % #{}) shape-id))
|
||||
(update :children dissoc shape-id))
|
||||
index))
|
||||
|
||||
(defn- index-add-page!
|
||||
[index {:keys [id name doc-id] page-index :index}]
|
||||
(-> index
|
||||
(assoc-in [:pages id] {:id id :name name :index page-index})
|
||||
(update :children update doc-id (fnil conj #{}) id)))
|
||||
|
||||
(defn- index-move-shape!
|
||||
[index shape-id {:keys [parent-id parent-table position page-id frame-id]}]
|
||||
(let [old-parent (get-in index [:shapes shape-id :parent-id])]
|
||||
(-> index
|
||||
(assoc-in [:shapes shape-id :parent-id] parent-id)
|
||||
(assoc-in [:shapes shape-id :parent-table] parent-table)
|
||||
(assoc-in [:shapes shape-id :position] position)
|
||||
(assoc-in [:shapes shape-id :frame-id] frame-id)
|
||||
(cond-> page-id (assoc-in [:shapes shape-id :page-id] page-id))
|
||||
(update :children update old-parent #(disj (or % #{}) shape-id))
|
||||
(update :children update parent-id (fnil conj #{}) shape-id))))
|
||||
|
||||
;; --- the columns that restate parenthood
|
||||
;;
|
||||
;; A shape carries `parent_id` and `frame_id`, and a container carries the
|
||||
;; ordered `shapes` list. All three restate what `IsChildOf` already says, and
|
||||
;; the cold projection writes them from the file, so this path has to keep
|
||||
;; them in step or a synced graph stops matching a rebuilt one.
|
||||
|
||||
(defn- shape-parent-id
|
||||
"The `parent_id` a shape's own column holds.
|
||||
|
||||
A top-level shape's parent in the file is the page's root frame, which the
|
||||
graph does not materialize, so `IsChildOf` points at the Page while the
|
||||
column holds `uuid/zero`."
|
||||
[parent-id parent-table]
|
||||
(if (= "Page" parent-table) uuid/zero parent-id))
|
||||
|
||||
(defn- frame-id-under
|
||||
"The `frame_id` a shape gets when its parent is `parent-id`.
|
||||
|
||||
Penpot's rule, from `app.common.files.changes` `:mov-objects`: the parent
|
||||
itself when the parent is a Frame, the parent's own frame otherwise."
|
||||
[index parent-id parent-table]
|
||||
(cond
|
||||
(= "Page" parent-table) uuid/zero
|
||||
(= "Frame" parent-table) parent-id
|
||||
:else (get-in index [:shapes parent-id :frame-id] uuid/zero)))
|
||||
|
||||
(defn- frame-id-updates
|
||||
"`[shape-id frame-id]` for a moved shape and everything that follows it.
|
||||
|
||||
A Frame keeps its descendants pointing at itself, so the walk stops there.
|
||||
Any other shape carries its subtree onto the new frame."
|
||||
[index shape-id frame-id]
|
||||
(into [[shape-id frame-id]]
|
||||
(when (not= "Frame" (get-in index [:shapes shape-id :table]))
|
||||
(mapcat #(frame-id-updates index % frame-id)
|
||||
(get-in index [:children shape-id] #{})))))
|
||||
|
||||
(defn- child-shapes-value
|
||||
"A container's stored `shapes` list, rebuilt from the index.
|
||||
|
||||
`IsChildOf.position` counts from the last entry of that list
|
||||
(`app.graph.projection.document/child-shape-ids` reverses it), so reversing the
|
||||
children ordered by position gives the list back."
|
||||
[index parent-id]
|
||||
(->> (get-in index [:children parent-id] #{})
|
||||
(sort-by #(get-in index [:shapes % :position] 0))
|
||||
reverse
|
||||
vec))
|
||||
|
||||
(defn- insert-position
|
||||
"The graph position the lowest of `k` shapes takes when they are inserted
|
||||
into a parent that already holds `n-before` children.
|
||||
|
||||
A container's stored `:shapes` list runs bottom to top, and the graph
|
||||
numbers children in Penpot z-order, so the two run opposite ways. An append
|
||||
to the stored list, which is what `:add-obj` does without an `:index`, is
|
||||
therefore position 0 and pushes every sibling up by one. The block occupies
|
||||
the result and the `k - 1` positions above it, the first shape highest."
|
||||
[n-before {:keys [index]} after-position]
|
||||
(cond
|
||||
(some? after-position) (long after-position)
|
||||
(some? index) (max 0 (- n-before (long index)))
|
||||
:else 0))
|
||||
|
||||
(defn- renumber-siblings
|
||||
"Shift `parent-id`'s children at or above `from` by `delta`.
|
||||
|
||||
Returns `[index statements]`. `except` names children the caller is placing
|
||||
itself."
|
||||
[index parent-id parent-table from delta except]
|
||||
(reduce
|
||||
(fn [[idx stmts] child-id]
|
||||
(let [pos (get-in idx [:shapes child-id :position])]
|
||||
(if (and (some? pos) (not (contains? except child-id)) (>= (long pos) (long from)))
|
||||
(let [pos' (+ (long pos) (long delta))]
|
||||
[(assoc-in idx [:shapes child-id :position] pos')
|
||||
(conj stmts (set-edge-position-statement
|
||||
{:from-table (get-in idx [:shapes child-id :table])
|
||||
:from-id child-id
|
||||
:to-table parent-table
|
||||
:to-id parent-id
|
||||
:position pos'}))])
|
||||
[idx stmts])))
|
||||
[index []]
|
||||
(vec (get-in index [:children parent-id] #{}))))
|
||||
|
||||
(defn- set-children-statements
|
||||
"Refresh the `shapes` column of every container in `parent-ids`.
|
||||
|
||||
A Page has no such column: its top-level shapes hang off a root frame the
|
||||
graph never materializes."
|
||||
[index parent-ids]
|
||||
(into []
|
||||
(comp (distinct)
|
||||
(keep (fn [parent-id]
|
||||
(let [table (get-in index [:shapes parent-id :table])]
|
||||
(when (contains? nodes/container-tables table)
|
||||
(set-node-attr-statement
|
||||
table parent-id :shapes
|
||||
(child-shapes-value index parent-id)))))))
|
||||
parent-ids))
|
||||
|
||||
(defn- mov-object-ids
|
||||
[shapes]
|
||||
(let [coll (cond
|
||||
(nil? shapes) []
|
||||
(sequential? shapes) shapes
|
||||
(uuid? shapes) [shapes]
|
||||
(map? shapes) (if-let [id (or (:id shapes) (get shapes "id"))]
|
||||
[id]
|
||||
[])
|
||||
:else [])]
|
||||
(into []
|
||||
(keep (fn [shape]
|
||||
(when shape
|
||||
(if (uuid? shape) shape (:id shape)))))
|
||||
coll)))
|
||||
|
||||
(defn- detach-shape
|
||||
"Take `shape-id` out of its current parent and close the gap it leaves.
|
||||
|
||||
Returns `[index statements]`. The edge itself is left alone: the caller
|
||||
either replaces it or deletes it."
|
||||
[index shape-id]
|
||||
(let [{:keys [parent-id parent-table position]} (get-in index [:shapes shape-id])
|
||||
index (update-in index [:children parent-id] #(disj (or % #{}) shape-id))
|
||||
[index stmts] (renumber-siblings index parent-id parent-table
|
||||
(inc (long (or position 0))) -1 #{})]
|
||||
[(assoc-in index [:shapes shape-id :position] nil) stmts]))
|
||||
|
||||
(defn- apply-mov-objects
|
||||
[index {:keys [shapes page-id] :as change}]
|
||||
(let [shape-ids (mov-object-ids shapes)
|
||||
parent (resolve-parent-for-add index
|
||||
(assoc change
|
||||
:frame-id (:parent-id change)
|
||||
:page-id page-id))]
|
||||
(cond
|
||||
(empty? shape-ids)
|
||||
{:index index :statements [] :applied? true}
|
||||
|
||||
(not parent)
|
||||
{:index index :statements [] :applied? false :reason :missing-parent}
|
||||
|
||||
:else
|
||||
(let [parent-id (:parent-id parent)
|
||||
parent-table (:parent-table parent)
|
||||
page-id' (or page-id
|
||||
(when (= parent-table "Page") parent-id)
|
||||
(get-in index [:shapes (first shape-ids) :page-id]))
|
||||
known (filterv #(get-in index [:shapes %]) shape-ids)
|
||||
old-parents (mapv #(get-in index [:shapes % :parent-id]) known)
|
||||
;; Penpot removes the shapes from wherever they were, then inserts
|
||||
;; the block into the target, so the target's width is measured
|
||||
;; after the removals.
|
||||
[index detach-stmts]
|
||||
(reduce (fn [[idx stmts] shape-id]
|
||||
(let [[idx' s] (detach-shape idx shape-id)]
|
||||
[idx' (into stmts s)]))
|
||||
[index []]
|
||||
known)
|
||||
n-before (count (get-in index [:children parent-id] #{}))
|
||||
after-pos (get-in index [:shapes (:after-shape change) :position])
|
||||
lowest (insert-position n-before change after-pos)
|
||||
k (count known)
|
||||
[index shift-stmts]
|
||||
(renumber-siblings index parent-id parent-table lowest k #{})]
|
||||
(loop [index index
|
||||
statements (into detach-stmts shift-stmts)
|
||||
entries (map-indexed vector known)]
|
||||
(if-let [[offset shape-id] (first entries)]
|
||||
(let [shape (get-in index [:shapes shape-id])
|
||||
position (+ lowest (- k 1 (long offset)))
|
||||
frame-id (frame-id-under index parent-id parent-table)
|
||||
frame-writes (frame-id-updates index shape-id frame-id)
|
||||
edge {:from-table (:table shape)
|
||||
:from-id shape-id
|
||||
:to-table parent-table
|
||||
:to-id parent-id
|
||||
:position position}
|
||||
moved? (not= parent-id (:parent-id shape))
|
||||
statements (-> statements
|
||||
(cond-> moved?
|
||||
(conj (delete-edge-statement
|
||||
{:from-table (:table shape)
|
||||
:from-id shape-id
|
||||
:to-table (:parent-table shape)
|
||||
:to-id (:parent-id shape)})))
|
||||
(conj (if moved?
|
||||
(create-edge-statement edge)
|
||||
(set-edge-position-statement edge))))
|
||||
;; The shape's own columns restate the edge, and the frame
|
||||
;; follows the whole subtree the shape carries with it.
|
||||
statements (if-not moved?
|
||||
statements
|
||||
(into (conj statements
|
||||
(set-node-attr-statement
|
||||
(:table shape) shape-id :parent-id
|
||||
(shape-parent-id parent-id parent-table)))
|
||||
(map (fn [[sid fid]]
|
||||
(set-node-attr-statement
|
||||
(get-in index [:shapes sid :table])
|
||||
sid :frame-id fid)))
|
||||
frame-writes))
|
||||
index (index-move-shape! index shape-id
|
||||
{:parent-id parent-id
|
||||
:parent-table parent-table
|
||||
:position position
|
||||
:frame-id frame-id
|
||||
:page-id page-id'})
|
||||
index (reduce (fn [idx [sid fid]]
|
||||
(assoc-in idx [:shapes sid :frame-id] fid))
|
||||
index
|
||||
frame-writes)]
|
||||
(recur index statements (rest entries)))
|
||||
{:index index
|
||||
:statements (into statements
|
||||
(set-children-statements index (conj old-parents parent-id)))
|
||||
:applied? true}))))))
|
||||
|
||||
(defn- index-remove-page!
|
||||
[index page-id]
|
||||
(let [doc-id (:doc-id index)]
|
||||
(-> index
|
||||
(update :pages dissoc page-id)
|
||||
(update :children update doc-id #(disj (or % #{}) page-id))
|
||||
(update :children dissoc page-id))))
|
||||
|
||||
(defn- mod-attrs-for-table
|
||||
[table]
|
||||
(disj (set (nodes/column-keys table)) :id))
|
||||
|
||||
(defn- apply-add-obj
|
||||
[index change]
|
||||
(let [{:keys [id obj page-id]} change
|
||||
table (shape-table obj)]
|
||||
(if-not table
|
||||
{:index index :statements [] :applied? false :reason :unsupported-shape-type}
|
||||
(let [parent (resolve-parent-for-add index change)]
|
||||
(if-not parent
|
||||
{:index index :statements [] :applied? false :reason :missing-parent}
|
||||
(let [parent-id (:parent-id parent)
|
||||
parent-table (:parent-table parent)
|
||||
n-before (count (get-in index [:children parent-id] #{}))
|
||||
position (insert-position n-before change nil)
|
||||
[index shift-stmts]
|
||||
(renumber-siblings index parent-id parent-table position 1 #{})
|
||||
;; The same denormalizations the cold projection performs, so
|
||||
;; a live-synced graph and a rebuilt one carry equal columns.
|
||||
resolved-page-id
|
||||
(or page-id
|
||||
(when (= parent-table "Page") parent-id)
|
||||
(get-in index [:shapes parent-id :page-id]))
|
||||
parent-ctx (get-in index [:shapes parent-id :component-ctx])
|
||||
shape (projection.document/denormalized-shape
|
||||
(assoc obj :id id) resolved-page-id parent-ctx)
|
||||
attrs (nodes/project-attrs table shape)
|
||||
edge {:from-table table
|
||||
:from-id id
|
||||
:to-table parent-table
|
||||
:to-id parent-id
|
||||
:position position}
|
||||
stmts (-> shift-stmts
|
||||
(conj (create-node-statement table attrs))
|
||||
(conj (create-edge-statement edge))
|
||||
(into (instance-of-statements table id (:component-id attrs))))
|
||||
index' (index-add-shape! index
|
||||
{:id id
|
||||
:name (:name attrs)
|
||||
:table table
|
||||
:parent-id parent-id
|
||||
:parent-table parent-table
|
||||
:position position
|
||||
:frame-id (:frame-id attrs)
|
||||
:component-ctx (projection.document/descend-component-ctx
|
||||
table shape parent-ctx)
|
||||
:page-id resolved-page-id})]
|
||||
{:index index'
|
||||
:statements (into stmts (set-children-statements index' [parent-id]))
|
||||
:applied? true}))))))
|
||||
|
||||
(defn- apply-mod-obj
|
||||
[index {:keys [id operations]}]
|
||||
(if-let [shape (get-in index [:shapes id])]
|
||||
(let [table (:table shape)
|
||||
syncable (mod-attrs-for-table table)
|
||||
set-ops (filter #(and (= :set (:type %))
|
||||
(contains? syncable (:attr %)))
|
||||
operations)]
|
||||
(if (empty? set-ops)
|
||||
{:index index :statements [] :applied? false :reason :unsupported-operations}
|
||||
(let [updates (into {} (map (juxt :attr :val) set-ops))
|
||||
statements
|
||||
(into (vec (for [[attr value] updates]
|
||||
(set-node-attr-statement table id attr value)))
|
||||
;; Relink when component-id is among the synced attrs.
|
||||
(when (contains? updates :component-id)
|
||||
(instance-of-statements table id (:component-id updates))))
|
||||
index' (reduce (fn [idx [attr value]]
|
||||
(assoc-in idx [:shapes id attr] value))
|
||||
index
|
||||
updates)]
|
||||
{:index index'
|
||||
:statements statements
|
||||
:applied? true})))
|
||||
{:index index :statements [] :applied? false :reason :missing-shape}))
|
||||
|
||||
(defn- delete-order-deepest-first
|
||||
[children root-id]
|
||||
(letfn [(post-order [id]
|
||||
(into (mapcat post-order (get children id #{}))
|
||||
[id]))]
|
||||
(post-order root-id)))
|
||||
|
||||
(defn- apply-del-obj
|
||||
[index {:keys [id]}]
|
||||
(if-let [root (get-in index [:shapes id])]
|
||||
(let [to-delete (delete-order-deepest-first (:children index) id)
|
||||
statements
|
||||
(vec (mapcat (fn [shape-id]
|
||||
(let [{:keys [table parent-id parent-table]}
|
||||
(get-in index [:shapes shape-id])]
|
||||
[(delete-edge-statement
|
||||
{:from-table table
|
||||
:from-id shape-id
|
||||
:to-table parent-table
|
||||
:to-id parent-id})
|
||||
(delete-node-statement table shape-id)]))
|
||||
to-delete))
|
||||
index' (reduce index-remove-shape! index to-delete)
|
||||
;; Only the deleted subtree's own parent survives to be renumbered:
|
||||
;; every other parent in `to-delete` goes with it.
|
||||
[index' shift-stmts]
|
||||
(renumber-siblings index' (:parent-id root) (:parent-table root)
|
||||
(inc (long (or (:position root) 0))) -1 #{})]
|
||||
{:index index'
|
||||
:statements (-> statements
|
||||
(into shift-stmts)
|
||||
(into (set-children-statements index' [(:parent-id root)])))
|
||||
:applied? true})
|
||||
;; Penpot emits one :del-obj per selected shape; an earlier change in the
|
||||
;; same batch may have already removed this node (e.g. parent + child).
|
||||
{:index index :statements [] :applied? true}))
|
||||
|
||||
(defn- apply-add-page
|
||||
[index {:keys [id name page]}]
|
||||
(let [page-id (or id (:id page))
|
||||
page (or page {:id page-id :name name})
|
||||
page (nodes/project-attrs "Page" {:id page-id
|
||||
:name (or (:name page) "Page")
|
||||
:index (count (:pages index))})
|
||||
doc-id (:doc-id index)
|
||||
position (count (:pages index))
|
||||
edge {:from-table "Page"
|
||||
:from-id page-id
|
||||
:to-table "Document"
|
||||
:to-id doc-id
|
||||
:position position}]
|
||||
{:index (index-add-page! index
|
||||
{:id page-id
|
||||
:name (:name page)
|
||||
:index (:index page)
|
||||
:doc-id doc-id})
|
||||
:statements [(create-node-statement "Page" page)
|
||||
(create-edge-statement edge)]
|
||||
:applied? true}))
|
||||
|
||||
(defn- apply-del-page
|
||||
[index {:keys [id]}]
|
||||
(if (get-in index [:pages id])
|
||||
(let [shape-ids (into #{}
|
||||
(comp (filter #(= id (get-in index [:shapes % :page-id])))
|
||||
(filter #(= "Page" (get-in index [:shapes % :parent-table]))))
|
||||
(keys (:shapes index)))
|
||||
del-shapes
|
||||
(reduce (fn [acc shape-id]
|
||||
(let [result (apply-del-obj acc {:type :del-obj :id shape-id})]
|
||||
(if (:applied? result)
|
||||
(-> acc
|
||||
(assoc :index (:index result))
|
||||
(update :statements into (:statements result)))
|
||||
acc)))
|
||||
{:index index :statements []}
|
||||
shape-ids)
|
||||
statements
|
||||
(conj (:statements del-shapes)
|
||||
(delete-edge-statement {:from-table "Page"
|
||||
:from-id id
|
||||
:to-table "Document"
|
||||
:to-id (:doc-id index)})
|
||||
(delete-node-statement "Page" id))]
|
||||
{:index (-> (:index del-shapes) (index-remove-page! id))
|
||||
:statements statements
|
||||
:applied? true})
|
||||
{:index index :statements [] :applied? false :reason :missing-page}))
|
||||
|
||||
(defn- apply-mod-page
|
||||
[index {:keys [id name]}]
|
||||
(if (and (string? name) (get-in index [:pages id]))
|
||||
{:index (assoc-in index [:pages id :name] name)
|
||||
:statements [(set-page-name-statement id name)]
|
||||
:applied? true}
|
||||
{:index index :statements [] :applied? false :reason :unsupported-page-change}))
|
||||
|
||||
(defn- component-syncable-attrs
|
||||
"Projected Component columns that sync may SET (everything but :id)."
|
||||
[]
|
||||
(disj (set (nodes/column-keys "Component")) :id))
|
||||
|
||||
(defn- component-attrs-from-change
|
||||
"Build CREATE attrs for `:add-component` (objects are not projected)."
|
||||
[{:keys [id name path main-instance-id main-instance-page
|
||||
annotation variant-id variant-properties]}]
|
||||
(cond-> {:id id
|
||||
:name (or name "Component")
|
||||
:path (or path "")
|
||||
:main-instance-id main-instance-id
|
||||
:main-instance-page main-instance-page}
|
||||
(some? annotation) (assoc :annotation annotation)
|
||||
(some? variant-id) (assoc :variant-id variant-id)
|
||||
(seq variant-properties) (assoc :variant-properties variant-properties)))
|
||||
|
||||
(defn- apply-add-component
|
||||
[index {:keys [id] :as change}]
|
||||
(if (get-in index [:components id])
|
||||
{:index index :statements [] :applied? true}
|
||||
(let [doc-id (:doc-id index)
|
||||
position (count (:components index))
|
||||
attrs (nodes/project-attrs "Component" (component-attrs-from-change change))
|
||||
edge {:from-table "Component"
|
||||
:from-id id
|
||||
:to-table "Document"
|
||||
:to-id doc-id
|
||||
:position position}]
|
||||
{:index (index-add-component! index
|
||||
{:id id
|
||||
:name (:name attrs)
|
||||
:doc-id doc-id})
|
||||
:statements [(create-node-statement "Component" attrs)
|
||||
(create-edge-statement edge)]
|
||||
:applied? true})))
|
||||
|
||||
(defn- apply-mod-component
|
||||
"Update projected Component attrs from a `:mod-component` change.
|
||||
|
||||
Nil optional values clear the property (Penpot dissocs them). `:objects`
|
||||
is never projected — shape trees live on pages."
|
||||
[index {:keys [id] :as change}]
|
||||
(let [syncable (component-syncable-attrs)
|
||||
sets (into {}
|
||||
(keep (fn [[k v]]
|
||||
(when (and (contains? syncable k) (some? v))
|
||||
[k v])))
|
||||
(dissoc change :type :id :objects))
|
||||
removes (into []
|
||||
(keep (fn [[k v]]
|
||||
(when (and (contains? syncable k) (nil? v))
|
||||
k)))
|
||||
(dissoc change :type :id :objects))
|
||||
stmts (into (mapv (fn [[k v]]
|
||||
(set-node-attr-statement "Component" id k v))
|
||||
sets)
|
||||
(map #(remove-node-attr-statement "Component" id %) removes))
|
||||
index' (if (get-in index [:components id])
|
||||
(cond-> index
|
||||
(contains? sets :name)
|
||||
(assoc-in [:components id :name] (:name sets)))
|
||||
(assoc-in index [:components id]
|
||||
{:id id
|
||||
:name (:name sets)
|
||||
:deleted false}))]
|
||||
(if (empty? stmts)
|
||||
{:index index :statements [] :applied? true}
|
||||
{:index index' :statements stmts :applied? true})))
|
||||
|
||||
(defn- apply-del-component
|
||||
[index {:keys [id skip-undelete?]}]
|
||||
(cond
|
||||
(not (get-in index [:components id]))
|
||||
{:index index :statements [] :applied? true}
|
||||
|
||||
skip-undelete?
|
||||
{:index (index-remove-component! index id)
|
||||
:statements [(delete-edge-statement {:from-table "Component"
|
||||
:from-id id
|
||||
:to-table "Document"
|
||||
:to-id (:doc-id index)})
|
||||
(delete-node-statement "Component" id)]
|
||||
:applied? true}
|
||||
|
||||
:else
|
||||
{:index (assoc-in index [:components id :deleted] true)
|
||||
:statements [(set-node-attr-statement "Component" id :deleted true)]
|
||||
:applied? true}))
|
||||
|
||||
(defn- apply-restore-component
|
||||
[index {:keys [id page-id]}]
|
||||
(let [stmts (cond-> [(set-node-attr-statement "Component" id :deleted false)]
|
||||
page-id
|
||||
(conj (set-node-attr-statement "Component" id :main-instance-page page-id)))
|
||||
index (if (get-in index [:components id])
|
||||
(-> index
|
||||
(assoc-in [:components id :deleted] false)
|
||||
(cond-> page-id
|
||||
(assoc-in [:components id :main-instance-page] page-id)))
|
||||
(assoc-in index [:components id]
|
||||
{:id id :name nil :deleted false}))]
|
||||
{:index index :statements stmts :applied? true}))
|
||||
|
||||
(defn- apply-purge-component
|
||||
[index {:keys [id]}]
|
||||
(if-not (get-in index [:components id])
|
||||
;; Still attempt delete in case the node exists but was not indexed.
|
||||
{:index index
|
||||
:statements [(delete-edge-statement {:from-table "Component"
|
||||
:from-id id
|
||||
:to-table "Document"
|
||||
:to-id (:doc-id index)})
|
||||
(delete-node-statement "Component" id)]
|
||||
:applied? true}
|
||||
{:index (index-remove-component! index id)
|
||||
:statements [(delete-edge-statement {:from-table "Component"
|
||||
:from-id id
|
||||
:to-table "Document"
|
||||
:to-id (:doc-id index)})
|
||||
(delete-node-statement "Component" id)]
|
||||
:applied? true}))
|
||||
|
||||
(defn- apply-change
|
||||
[index change]
|
||||
(case (:type change)
|
||||
:add-obj (apply-add-obj index change)
|
||||
:mod-obj (apply-mod-obj index change)
|
||||
:del-obj (apply-del-obj index change)
|
||||
:add-page (apply-add-page index change)
|
||||
:del-page (apply-del-page index change)
|
||||
:mod-page (apply-mod-page index change)
|
||||
:mov-objects (apply-mov-objects index change)
|
||||
:add-component (apply-add-component index change)
|
||||
:mod-component (apply-mod-component index change)
|
||||
:del-component (apply-del-component index change)
|
||||
:restore-component (apply-restore-component index change)
|
||||
:purge-component (apply-purge-component index change)
|
||||
{:index index :statements [] :applied? false :reason :unsupported-type}))
|
||||
|
||||
(defn apply-changes!
|
||||
"Apply Penpot `changes` to an open Ladybug `conn` and return the updated index.
|
||||
|
||||
Returns `{:index ... :revn ... :applied [...] :skipped [...]}`."
|
||||
[^Connection conn index changes revn]
|
||||
(when (> (long revn) (:revn index))
|
||||
(l/wrn :hint "graph sync revn gap"
|
||||
:file-id (str (:file-id index))
|
||||
:index-revn (:revn index)
|
||||
:change-revn revn))
|
||||
(loop [index index
|
||||
applied []
|
||||
skipped []
|
||||
stmts []
|
||||
changes (seq changes)]
|
||||
(if-let [change (first changes)]
|
||||
(let [{:keys [index statements applied? reason]}
|
||||
(apply-change index change)]
|
||||
(recur index
|
||||
(cond-> applied applied? (conj (:type change)))
|
||||
(cond-> skipped (not applied?) (conj {:type (:type change) :reason reason}))
|
||||
(cond-> stmts applied? (into statements))
|
||||
(rest changes)))
|
||||
(let [final-stmts (cond-> stmts
|
||||
(and (seq applied) (:doc-id index))
|
||||
(conj (set-document-revision-statement (:doc-id index) revn)))
|
||||
index' (if (seq applied)
|
||||
(assoc index :revn (long revn))
|
||||
index)]
|
||||
(when (seq final-stmts)
|
||||
(ladybug/exec-on-connection! conn final-stmts))
|
||||
{:index index'
|
||||
:revn (if (seq applied) (long revn) (:revn index'))
|
||||
:applied applied
|
||||
:skipped skipped}))))
|
||||
|
||||
(defn supported-change?
|
||||
[change]
|
||||
(contains? supported-change-types (:type change)))
|
||||
+331
-24
@@ -16,6 +16,7 @@
|
||||
[app.common.files.changes :as cfc]
|
||||
[app.common.files.repair :as cfr]
|
||||
[app.common.files.validate :as cfv]
|
||||
[app.common.json :as json]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.time :as ct]
|
||||
@@ -25,6 +26,7 @@
|
||||
[app.db :as db]
|
||||
[app.features.file-migrations :as feat.fmig]
|
||||
[app.http.session :as session]
|
||||
[app.redis :as rds]
|
||||
[app.rpc.commands.auth :as auth]
|
||||
[app.rpc.commands.files-create :refer [create-file]]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
@@ -37,6 +39,7 @@
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.template :as tmpl]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
[datoteka.io :as io]
|
||||
[emoji.core :as emj]
|
||||
[integrant.core :as ig]
|
||||
@@ -51,21 +54,66 @@
|
||||
;; INDEX
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(def ^:private max-export-jobs 200)
|
||||
|
||||
(defn- scan-export-job-keys
|
||||
"Note: no index for now, get them all and filter"
|
||||
[conn pattern]
|
||||
(loop [cursor "0"
|
||||
found []]
|
||||
(let [[cursor keys] (rds/scan conn cursor pattern max-export-jobs)
|
||||
found (into found keys)]
|
||||
(if (or (nil? cursor)
|
||||
(= "0" cursor)
|
||||
(>= (count found) max-export-jobs))
|
||||
(into [] (take max-export-jobs) found)
|
||||
(recur cursor found)))))
|
||||
|
||||
(defn- get-export-jobs
|
||||
[cfg job-id]
|
||||
(let [filtered? (not (str/empty-or-nil? job-id))
|
||||
job-uuid (when filtered? (parse-uuid job-id))]
|
||||
(if (and filtered? (nil? job-uuid))
|
||||
[]
|
||||
(try
|
||||
(let [pattern (str "penpot.exporter." (cf/get :tenant) ".job." (or job-uuid "*"))]
|
||||
(->> (rds/run! cfg (fn [{:keys [::rds/conn]}]
|
||||
(->> (scan-export-job-keys conn pattern)
|
||||
(mapv (fn [key] (rds/hget conn key "data"))))))
|
||||
(keep (fn [blob]
|
||||
(try
|
||||
(t/decode-str blob)
|
||||
(catch Throwable _ nil))))
|
||||
(sort-by :created-at #(compare %2 %1))
|
||||
;; The exporter stores instants as epoch millis.
|
||||
(map (fn [{:keys [created-at ended-at] :as job}]
|
||||
(-> job
|
||||
(assoc :created-at (some-> created-at ct/inst (ct/format-inst :rfc1123)))
|
||||
(assoc :ended-at (some-> ended-at ct/inst (ct/format-inst :rfc1123))))))
|
||||
(vec)))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unable to read export jobs" :cause cause)
|
||||
[])))))
|
||||
|
||||
(defn index-handler
|
||||
[cfg request]
|
||||
(let [profile-id (::session/profile-id request)
|
||||
offset (clock/get-offset profile-id)
|
||||
profile (profile/get-profile cfg profile-id)]
|
||||
profile (profile/get-profile cfg profile-id)
|
||||
job-filter (some-> request :params :job-id str/trim)]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/html"}
|
||||
::yres/body (-> (io/resource "app/templates/debug.tmpl")
|
||||
(tmpl/render {:version (:full cf/version)
|
||||
:profile profile
|
||||
:graph-enabled (contains? cf/flags :graph)
|
||||
:current-clock ct/*clock*
|
||||
:current-offset (if offset
|
||||
(ct/format-duration offset)
|
||||
"NO OFFSET")
|
||||
:current-time (ct/format-inst (ct/now) :http)
|
||||
:export-jobs (get-export-jobs cfg job-filter)
|
||||
:export-job-filter job-filter
|
||||
:supported-features cfeat/supported-features}))}))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
@@ -318,11 +366,17 @@
|
||||
(if clone?
|
||||
(let [profile (profile/get-profile pool profile-id)
|
||||
project-id (:default-project-id profile)
|
||||
team (teams/get-team pool
|
||||
:profile-id profile-id
|
||||
:project-id project-id)
|
||||
cfg (assoc cfg
|
||||
::bfc/overwrite false
|
||||
::bfc/profile-id profile-id
|
||||
::bfc/project-id project-id
|
||||
::bfc/input path)]
|
||||
::bfc/team-id (:id team)
|
||||
::bfc/input path
|
||||
::bfc/import-max-object-size (cf/get :binfile-import-max-object-size)
|
||||
::bfc/import-max-zip-entries (cf/get :binfile-import-max-zip-entries))]
|
||||
(bf.v3/import-files! cfg)
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/plain"}
|
||||
@@ -334,6 +388,226 @@
|
||||
"content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
|
||||
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; GRAPH (flag: :graph)
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; `app.graph.*` resolves at call time, never at the top of this namespace.
|
||||
;; `app.graph.ladybug` imports `com.ladybugdb.*`, so requiring it links the
|
||||
;; Ladybug native library into the JVM, and this namespace loads on every
|
||||
;; backend boot. The routes below are registered only under the `:graph` flag,
|
||||
;; so with the flag off nothing resolves and no native code loads.
|
||||
|
||||
(defn- graph-export-file
|
||||
"Path of a freshly projected graph for `file-id`."
|
||||
[cfg file-id]
|
||||
(let [ingest-file! (requiring-resolve 'app.graph.ingest/ingest-file!)
|
||||
{:keys [db-path]} (ingest-file! cfg file-id :skip-stats? true)]
|
||||
(when-not (fs/exists? db-path)
|
||||
(ex/raise :type :internal
|
||||
:code :graph-file-not-found
|
||||
:hint "graph database file missing after ingest"
|
||||
:file-id (str file-id)
|
||||
:db-path db-path))
|
||||
db-path))
|
||||
|
||||
(defn- graph-export-session
|
||||
"Path of a snapshot of the caller's live in-memory graph for `file-id`."
|
||||
[profile-id file-id]
|
||||
(let [session-info (requiring-resolve 'app.graph.debug/session-info)
|
||||
export-session-database! (requiring-resolve 'app.graph.debug/export-session-database!)
|
||||
info (session-info profile-id)]
|
||||
(when-not info
|
||||
(ex/raise :type :not-found
|
||||
:code :graph-session-not-loaded
|
||||
:hint "no in-memory graph is loaded; load one first, or use source=file"))
|
||||
(when-not (= file-id (:file-id info))
|
||||
(ex/raise :type :validation
|
||||
:code :graph-session-file-mismatch
|
||||
:hint "the loaded session holds a different file"
|
||||
:requested (str file-id)
|
||||
:loaded (str (:file-id info))))
|
||||
(export-session-database! profile-id)))
|
||||
|
||||
(defn graph-export-handler
|
||||
"Stream a Ladybug `.lbug` database for a file.
|
||||
|
||||
`source=file` (default) projects the file afresh from the database — the
|
||||
reproducible artifact. `source=session` snapshots the caller's live
|
||||
in-memory console graph instead, which live-sync may have moved away from a
|
||||
fresh projection; taking that away to query it elsewhere is the whole point
|
||||
of asking for it. Synchronous on each request."
|
||||
[cfg {:keys [params] :as request}]
|
||||
(let [file-id (some-> params :file-id parse-uuid)
|
||||
source (or (some-> params :source str/lower) "file")]
|
||||
(when-not file-id
|
||||
(ex/raise :type :validation
|
||||
:code :missing-arguments
|
||||
:hint "missing file-id"))
|
||||
(when-not (contains? #{"file" "session"} source)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-arguments
|
||||
:hint "source must be 'file' or 'session'"
|
||||
:source source))
|
||||
|
||||
(let [session? (= "session" source)
|
||||
db-path (if session?
|
||||
(graph-export-session (::session/profile-id request) file-id)
|
||||
(graph-export-file cfg file-id))]
|
||||
{::yres/status 200
|
||||
;; A session export is a temp file this request owns; deleting it on
|
||||
;; close would race the streaming body, so it is left for the OS temp
|
||||
;; sweep. A file export is the canonical per-file database and is meant
|
||||
;; to persist.
|
||||
::yres/body (io/input-stream db-path)
|
||||
::yres/headers {"content-type" "application/octet-stream"
|
||||
"content-disposition"
|
||||
(str "attachment; filename=" file-id
|
||||
(when session? "-session") ".lbug")}})))
|
||||
|
||||
(defn- graph-console-response
|
||||
[data]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"}
|
||||
::yres/body (-> (io/resource "app/templates/graph-console.tmpl")
|
||||
(tmpl/render (assoc data :version (:full cf/version))))})
|
||||
|
||||
(defn graph-console-handler
|
||||
[_cfg {:keys [::session/profile-id]}]
|
||||
(let [console-context (requiring-resolve 'app.graph.debug/console-context)]
|
||||
(graph-console-response (console-context profile-id))))
|
||||
|
||||
(defn graph-load-handler
|
||||
[cfg {:keys [params ::session/profile-id]}]
|
||||
(let [file-id (some-> (:file-id params) parse-uuid)
|
||||
load-session! (requiring-resolve 'app.graph.debug/load-session!)]
|
||||
(when-not file-id
|
||||
(ex/raise :type :validation
|
||||
:code :missing-arguments
|
||||
:hint "missing file-id"))
|
||||
(load-session! cfg profile-id file-id)
|
||||
{::yres/status 302
|
||||
::yres/headers {"location" "/dbg/graph"}}))
|
||||
|
||||
(defn graph-unload-handler
|
||||
[_cfg {:keys [::session/profile-id]}]
|
||||
((requiring-resolve 'app.graph.debug/unload-session!) profile-id)
|
||||
{::yres/status 302
|
||||
::yres/headers {"location" "/dbg/graph"}})
|
||||
|
||||
(defn graph-reload-handler
|
||||
"Re-ingest the currently loaded file into the in-memory graph session."
|
||||
[cfg {:keys [::session/profile-id]}]
|
||||
(let [session-info (requiring-resolve 'app.graph.debug/session-info)
|
||||
load-session! (requiring-resolve 'app.graph.debug/load-session!)]
|
||||
(if-let [file-id (some-> (session-info profile-id) :file-id)]
|
||||
(do
|
||||
(load-session! cfg profile-id file-id)
|
||||
{::yres/status 302
|
||||
::yres/headers {"location" "/dbg/graph"}})
|
||||
(ex/raise :type :not-found
|
||||
:code :graph-session-not-loaded
|
||||
:hint "load a file graph before reloading"))))
|
||||
|
||||
(defn graph-sync-status-handler
|
||||
[_cfg {:keys [::session/profile-id]}]
|
||||
(if-let [status ((requiring-resolve 'app.graph.debug/sync-status) profile-id)]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (t/encode-str status {:type :json-verbose})}
|
||||
{::yres/status 404
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (t/encode-str {:error "no-session"} {:type :json-verbose})}))
|
||||
|
||||
(defn graph-data-handler
|
||||
"Export the in-memory session graph as plain JSON (not transit) for the
|
||||
G6 graph view embedded in the console page."
|
||||
[_cfg {:keys [::session/profile-id]}]
|
||||
(if-let [data ((requiring-resolve 'app.graph.debug/export-graph-data!) profile-id)]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (json/encode data)}
|
||||
{::yres/status 404
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (json/encode {:error "no-session"})}))
|
||||
|
||||
(def ^:private sql:graph-files
|
||||
"select t.id as team_id, t.name as team_name,
|
||||
p.id as project_id, p.name as project_name,
|
||||
f.id as file_id, f.name as file_name
|
||||
from team as t
|
||||
join team_profile_rel as tpr on (tpr.team_id = t.id)
|
||||
join project as p on (p.team_id = t.id)
|
||||
join file as f on (f.project_id = p.id)
|
||||
where tpr.profile_id = ?
|
||||
and t.deleted_at is null
|
||||
and p.deleted_at is null
|
||||
and f.deleted_at is null
|
||||
order by t.name, p.name, f.name
|
||||
limit 500")
|
||||
|
||||
(defn- graph-files-tree
|
||||
[rows]
|
||||
(->> (group-by (juxt :team-id :team-name) rows)
|
||||
(mapv (fn [[[team-id team-name] team-rows]]
|
||||
{:id (str team-id)
|
||||
:name team-name
|
||||
:projects
|
||||
(->> (group-by (juxt :project-id :project-name) team-rows)
|
||||
(mapv (fn [[[project-id project-name] project-rows]]
|
||||
{:id (str project-id)
|
||||
:name project-name
|
||||
:files (mapv (fn [{:keys [file-id file-name]}]
|
||||
{:id (str file-id) :name file-name})
|
||||
project-rows)}))
|
||||
(sort-by :name)
|
||||
(vec))}))
|
||||
(sort-by :name)
|
||||
(vec)))
|
||||
|
||||
(defn graph-files-handler
|
||||
"List teams -> projects -> files reachable by the current profile, as
|
||||
plain JSON for the graph console file tree."
|
||||
[{:keys [::db/pool]} {:keys [::session/profile-id]}]
|
||||
(let [rows (db/exec! pool [sql:graph-files profile-id])]
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (json/encode {:teams (graph-files-tree rows)})}))
|
||||
|
||||
(defn- json-request?
|
||||
[request]
|
||||
(some-> request
|
||||
(yreq/get-header "accept")
|
||||
(str/includes? "application/json")))
|
||||
|
||||
(defn graph-query-handler
|
||||
[_cfg {:keys [params ::session/profile-id] :as request}]
|
||||
(let [query (:query params)
|
||||
query-session! (requiring-resolve 'app.graph.debug/query-session!)
|
||||
console-context (requiring-resolve 'app.graph.debug/console-context)]
|
||||
(try
|
||||
(let [result (query-session! profile-id query)]
|
||||
(if (json-request? request)
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (t/encode-str {:query query
|
||||
:query-result result}
|
||||
{:type :json-verbose})}
|
||||
(graph-console-response (console-context profile-id
|
||||
:query query
|
||||
:query-result result))))
|
||||
(catch Throwable e
|
||||
(let [error (or (:hint (ex-data e)) (ex-message e))]
|
||||
(if (json-request? request)
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yres/body (t/encode-str {:query query :error error}
|
||||
{:type :json-verbose})}
|
||||
(graph-console-response (console-context profile-id
|
||||
:query query
|
||||
:error error))))))))
|
||||
|
||||
(defn import-handler
|
||||
[{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}]
|
||||
(when-not (contains? params :file)
|
||||
@@ -358,7 +632,10 @@
|
||||
::bfc/profile-id profile-id
|
||||
::bfc/project-id project-id
|
||||
::bfc/input path
|
||||
::bfc/features (cfeat/get-team-enabled-features cf/flags team))]
|
||||
::bfc/team-id (:id team)
|
||||
::bfc/features (cfeat/get-team-enabled-features cf/flags team)
|
||||
::bfc/import-max-object-size (cf/get :binfile-import-max-object-size)
|
||||
::bfc/import-max-zip-entries (cf/get :binfile-import-max-zip-entries))]
|
||||
|
||||
(if (= format :binfile-v3)
|
||||
(bf.v3/import-files! cfg)
|
||||
@@ -626,7 +903,12 @@
|
||||
(letfn [(handle-error [cause]
|
||||
(when-let [data (ex-data cause)]
|
||||
(when (= :validation (:type data))
|
||||
(str "Error: " (or (:hint data) (ex-message cause)) "\n"))))]
|
||||
(let [hint (or (:hint data) (ex-message cause))
|
||||
explain (ex/explain data)]
|
||||
(str "Error: " hint
|
||||
(when (and explain (not (str/includes? hint explain)))
|
||||
(str "\n" explain))
|
||||
"\n")))))]
|
||||
{:name ::errors
|
||||
:compile
|
||||
(fn [& _params]
|
||||
@@ -646,26 +928,51 @@
|
||||
(assert (db/pool? (::db/pool params)) "expected a valid database pool")
|
||||
(assert (session/manager? (::session/manager params)) "expected a valid session manager"))
|
||||
|
||||
(defn- graph-action-routes
|
||||
[cfg]
|
||||
[["/graph-export" {:handler (partial graph-export-handler cfg)}]
|
||||
["/graph-load" {:handler (partial graph-load-handler cfg)}]
|
||||
["/graph-query" {:handler (partial graph-query-handler cfg)}]
|
||||
["/graph-unload" {:handler (partial graph-unload-handler cfg)}]
|
||||
["/graph-reload" {:handler (partial graph-reload-handler cfg)}]
|
||||
["/graph-sync-status" {:handler (partial graph-sync-status-handler cfg)}]
|
||||
["/graph-data" {:handler (partial graph-data-handler cfg)}]
|
||||
["/graph-files" {:handler (partial graph-files-handler cfg)}]])
|
||||
|
||||
(defmethod ig/init-key ::routes
|
||||
[_ {:keys [::db/pool] :as cfg}]
|
||||
[["/readyz" {:handler (partial health-handler cfg)}]
|
||||
["/dbg" {:middleware [[session/authz cfg]
|
||||
[with-authorization pool]]}
|
||||
["" {:handler (partial index-handler cfg)}]
|
||||
["/health" {:handler (partial health-handler cfg)}]
|
||||
["/changelog" {:handler (partial changelog-handler cfg)}]
|
||||
["/error/:id" {:handler (partial error-handler cfg)}]
|
||||
["/error" {:handler (partial error-list-handler cfg)}]
|
||||
["/actions" {:middleware [[errors]]}
|
||||
["/set-virtual-clock"
|
||||
{:handler (partial set-virtual-clock cfg)}]
|
||||
["/resend-email-verification"
|
||||
{:handler (partial resend-email-notification cfg)}]
|
||||
["/handle-team-features"
|
||||
{:handler (partial handle-team-features cfg)}]
|
||||
["/file-export" {:handler (partial export-handler cfg)}]
|
||||
["/file-import" {:handler (partial import-handler cfg)}]
|
||||
["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]
|
||||
["/file-validate" {:handler (partial validate-file cfg)}]
|
||||
["/file-repair" {:handler (partial repair-file cfg)}]]]])
|
||||
;; The graph routes are registered only under the `:graph` flag. Left
|
||||
;; unregistered they 404, and nothing ever resolves `app.graph.*`. The `/dbg`
|
||||
;; admin gate is unchanged: it covers the graph routes exactly as before.
|
||||
(let [graph? (contains? cf/flags :graph)
|
||||
actions (cond-> ["/actions" {:middleware [[errors]]}
|
||||
["/set-virtual-clock"
|
||||
{:handler (partial set-virtual-clock cfg)}]
|
||||
["/resend-email-verification"
|
||||
{:handler (partial resend-email-notification cfg)}]
|
||||
["/handle-team-features"
|
||||
{:handler (partial handle-team-features cfg)}]
|
||||
["/file-export" {:handler (partial export-handler cfg)}]
|
||||
["/file-import" {:handler (partial import-handler cfg)}]
|
||||
["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]
|
||||
["/file-validate" {:handler (partial validate-file cfg)}]
|
||||
["/file-repair" {:handler (partial repair-file cfg)}]]
|
||||
graph? (into (graph-action-routes cfg)))
|
||||
dbg (cond-> ["/dbg" {:middleware [[session/authz cfg]
|
||||
[with-authorization pool]]}
|
||||
["" {:handler (partial index-handler cfg)}]
|
||||
["/health" {:handler (partial health-handler cfg)}]
|
||||
["/changelog" {:handler (partial changelog-handler cfg)}]
|
||||
["/error/:id" {:handler (partial error-handler cfg)}]
|
||||
["/error" {:handler (partial error-list-handler cfg)}]
|
||||
actions]
|
||||
graph? (conj ["/graph" {:handler (partial graph-console-handler cfg)}]))]
|
||||
(when graph?
|
||||
;; With the flag on, the Ladybug native library belongs to this process,
|
||||
;; so load it here. A missing or unusable library then fails the boot
|
||||
;; instead of the first console request.
|
||||
(require 'app.graph.debug 'app.graph.ingest))
|
||||
|
||||
[["/readyz" {:handler (partial health-handler cfg)}]
|
||||
dbg]))
|
||||
|
||||
@@ -71,12 +71,9 @@
|
||||
|
||||
(defmethod handle-error :rate-limit
|
||||
[err _ _]
|
||||
(let [data (ex-data err)]
|
||||
(let [headers (-> err ex-data ::http/headers)]
|
||||
{::yres/status 429
|
||||
::yres/body {:type :rate-limit
|
||||
:code (:code data)
|
||||
:hint (:hint data)
|
||||
:ttl (:ttl data)}}))
|
||||
::yres/headers headers}))
|
||||
|
||||
(defmethod handle-error :concurrency-limit
|
||||
[err _ _]
|
||||
|
||||
@@ -209,7 +209,7 @@
|
||||
[:enum
|
||||
"customer_service"
|
||||
"low_quality"
|
||||
"missing_feature"
|
||||
"missing_features"
|
||||
"other"
|
||||
"switched_service"
|
||||
"too_complex"
|
||||
|
||||
@@ -288,7 +288,9 @@
|
||||
|
||||
::http.debug/routes
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::rds/pool (ig/ref ::rds/pool)
|
||||
::session/manager (ig/ref ::session/manager)
|
||||
::mbus/msgbus (ig/ref ::mbus/msgbus)
|
||||
::sto/storage (ig/ref ::sto/storage)
|
||||
::setup/props (ig/ref ::setup/props)}
|
||||
|
||||
@@ -398,6 +400,8 @@
|
||||
|
||||
:delete-object
|
||||
(ig/ref :app.tasks.delete-object/handler)
|
||||
:demo-purge
|
||||
(ig/ref :app.tasks.demo-purge/handler)
|
||||
:process-webhook-event
|
||||
(ig/ref ::webhooks/process-event-handler)
|
||||
:run-webhook
|
||||
@@ -435,6 +439,9 @@
|
||||
:app.tasks.delete-object/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
:app.tasks.demo-purge/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
:app.tasks.file-gc/handler
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
@@ -245,7 +245,7 @@
|
||||
[:enum
|
||||
"customer_service"
|
||||
"low_quality"
|
||||
"missing_feature"
|
||||
"missing_features"
|
||||
"other"
|
||||
"switched_service"
|
||||
"too_complex"
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
io.lettuce.core.api.sync.RedisScriptingCommands
|
||||
io.lettuce.core.codec.RedisCodec
|
||||
io.lettuce.core.codec.StringCodec
|
||||
io.lettuce.core.KeyScanCursor
|
||||
io.lettuce.core.KeyValue
|
||||
io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
|
||||
io.lettuce.core.pubsub.RedisPubSubListener
|
||||
@@ -40,6 +41,8 @@
|
||||
io.lettuce.core.RedisURI
|
||||
io.lettuce.core.resource.ClientResources
|
||||
io.lettuce.core.resource.DefaultClientResources
|
||||
io.lettuce.core.ScanArgs
|
||||
io.lettuce.core.ScanCursor
|
||||
io.lettuce.core.ScriptOutputType
|
||||
io.lettuce.core.SetArgs
|
||||
io.netty.channel.nio.NioEventLoopGroup
|
||||
@@ -71,6 +74,8 @@
|
||||
(-blpop [_ timeout keys])
|
||||
(-eval [_ script])
|
||||
(-get [_ key])
|
||||
(-scan [_ cursor pattern limit])
|
||||
(-hget [_ key field])
|
||||
(-set [_ key val args])
|
||||
(-del [_ key-or-keys])
|
||||
(-ping [_]))
|
||||
@@ -205,6 +210,20 @@
|
||||
(assert (string? key) "key expected to be string")
|
||||
(.get cmd ^String key))
|
||||
|
||||
(-scan [_ cursor pattern limit]
|
||||
(let [args (-> (ScanArgs.)
|
||||
(.match ^String pattern)
|
||||
(.limit (long limit)))
|
||||
result (.scan cmd
|
||||
^ScanCursor (ScanCursor/of ^String cursor)
|
||||
^ScanArgs args)]
|
||||
(MapEntry/create
|
||||
(.getCursor ^KeyScanCursor result)
|
||||
(vec (.getKeys ^KeyScanCursor result)))))
|
||||
|
||||
(-hget [_ key field]
|
||||
(.hget cmd ^String key ^String field))
|
||||
|
||||
(-set [_ key val args]
|
||||
(.set cmd
|
||||
^String key
|
||||
@@ -345,6 +364,26 @@
|
||||
(l/err :hint "timeout on get redis key" :key key :cause cause)
|
||||
nil)))
|
||||
|
||||
(defn scan
|
||||
[conn cursor pattern limit]
|
||||
(assert (string? cursor) "cursor must be string instance")
|
||||
(assert (string? pattern) "pattern must be string instance")
|
||||
(try
|
||||
(-scan conn cursor pattern limit)
|
||||
(catch RedisCommandTimeoutException cause
|
||||
(l/err :hint "timeout on scan" :pattern pattern :cause cause)
|
||||
nil)))
|
||||
|
||||
(defn hget
|
||||
[conn key field]
|
||||
(assert (string? key) "key must be string instance")
|
||||
(assert (string? field) "field must be string instance")
|
||||
(try
|
||||
(-hget conn key field)
|
||||
(catch RedisCommandTimeoutException cause
|
||||
(l/err :hint "timeout on hget" :key key :cause cause)
|
||||
nil)))
|
||||
|
||||
(defn set
|
||||
([conn key val]
|
||||
(set conn key val nil))
|
||||
|
||||
@@ -400,6 +400,7 @@
|
||||
'app.rpc.commands.management
|
||||
'app.rpc.commands.media
|
||||
'app.rpc.commands.nitrate
|
||||
'app.rpc.commands.plugins
|
||||
'app.rpc.commands.profile
|
||||
'app.rpc.commands.projects
|
||||
'app.rpc.commands.search
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
(ns app.rpc.commands.auth
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.auth.login-lockout :as login-lockout]
|
||||
[app.auth.oidc :as oidc]
|
||||
[app.auth.passwords :as passwords]
|
||||
[app.common.data :as d]
|
||||
@@ -88,21 +87,9 @@
|
||||
(ex/raise :type :restriction
|
||||
:code :profile-blocked
|
||||
:hint "profile is marked as blocked"))
|
||||
(let [result (login-lockout/locked? cfg (:id profile))]
|
||||
(when (:locked? result)
|
||||
(ex/raise :type :rate-limit
|
||||
:code :account-locked
|
||||
:hint "account locked due to too many failed login attempts"
|
||||
:ttl (:ttl result))))
|
||||
(when-not (check-password cfg profile password)
|
||||
(let [result (login-lockout/record-failed-attempt! cfg (:id profile))]
|
||||
(if (and result (:locked? result))
|
||||
(ex/raise :type :rate-limit
|
||||
:code :account-locked
|
||||
:hint "account locked due to too many failed login attempts"
|
||||
:ttl (:ttl result))
|
||||
(ex/raise :type :validation
|
||||
:code :wrong-credentials))))
|
||||
(ex/raise :type :validation
|
||||
:code :wrong-credentials))
|
||||
(when-let [deleted-at (:deleted-at profile)]
|
||||
(when (ct/is-after? (ct/now) deleted-at)
|
||||
(ex/raise :type :validation
|
||||
@@ -126,7 +113,6 @@
|
||||
{:invitation-token (:invitation-token params)}
|
||||
(assoc profile :is-admin (let [admins (cf/get :admins)]
|
||||
(contains? admins (:email profile)))))]
|
||||
(login-lockout/clear-attempts! cfg (:id profile))
|
||||
(-> response
|
||||
(rph/with-transform (session/create-fn cfg profile))
|
||||
(rph/with-meta {::audit/props (audit/profile->props profile)
|
||||
@@ -195,14 +181,11 @@
|
||||
(update-password [conn profile-id]
|
||||
(let [pwd (auth/derive-password password)]
|
||||
(db/update! conn :profile {:password pwd :is-active true} {:id profile-id})
|
||||
(db/get-by-id conn :profile profile-id)))]
|
||||
nil))]
|
||||
|
||||
(passwords/validate-password password)
|
||||
|
||||
(let [profile (some-> (validate-token token)
|
||||
(update-password conn))]
|
||||
(when profile
|
||||
(login-lockout/clear-attempts! cfg (:id profile))))
|
||||
(->> (validate-token token)
|
||||
(update-password conn))
|
||||
|
||||
nil))
|
||||
|
||||
|
||||
@@ -103,7 +103,9 @@
|
||||
(assoc ::bfc/project-id project-id)
|
||||
(assoc ::bfc/profile-id profile-id)
|
||||
(assoc ::bfc/team-id (:id team))
|
||||
(assoc ::bfc/name name))
|
||||
(assoc ::bfc/name name)
|
||||
(assoc ::bfc/import-max-object-size (cf/get :binfile-import-max-object-size))
|
||||
(assoc ::bfc/import-max-zip-entries (cf/get :binfile-import-max-zip-entries)))
|
||||
|
||||
input-path (:path file)
|
||||
owned? (some? upload-id)
|
||||
|
||||
@@ -231,8 +231,11 @@
|
||||
::sm/params schema:get-comment-threads}
|
||||
[cfg {:keys [::rpc/profile-id file-id share-id] :as params}]
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-comment-permissions! cfg profile-id file-id share-id)
|
||||
(get-comment-threads conn profile-id file-id))))
|
||||
(let [perms (files/check-comment-permissions! cfg profile-id file-id share-id)
|
||||
threads (get-comment-threads conn profile-id file-id)]
|
||||
(if (= :share-link (:type perms))
|
||||
(filterv #(contains? (:pages perms) (:page-id %)) threads)
|
||||
threads)))))
|
||||
|
||||
(defn- get-comment-threads-sql
|
||||
[where]
|
||||
@@ -329,9 +332,15 @@
|
||||
::sm/params schema:get-comment-thread}
|
||||
[cfg {:keys [::rpc/profile-id file-id id share-id] :as params}]
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-comment-permissions! cfg profile-id file-id share-id)
|
||||
(some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
|
||||
(decode-row)))))
|
||||
(let [perms (files/check-comment-permissions! cfg profile-id file-id share-id)
|
||||
thread (some-> (db/exec-one! conn [sql:get-comment-thread profile-id file-id id])
|
||||
(decode-row))]
|
||||
(when (and thread (= :share-link (:type perms)))
|
||||
(when-not (contains? (:pages perms) (:page-id thread))
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "not found")))
|
||||
thread))))
|
||||
|
||||
;; --- COMMAND: Retrieve Comments
|
||||
|
||||
@@ -348,8 +357,13 @@
|
||||
::sm/params schema:get-comments}
|
||||
[cfg {:keys [::rpc/profile-id thread-id share-id]}]
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(let [{:keys [file-id]} (get-comment-thread conn thread-id)]
|
||||
(files/check-comment-permissions! cfg profile-id file-id share-id)
|
||||
(let [{:keys [file-id page-id]} (get-comment-thread conn thread-id)
|
||||
perms (files/check-comment-permissions! cfg profile-id file-id share-id)]
|
||||
(when (and (= :share-link (:type perms))
|
||||
(not (contains? (:pages perms) page-id)))
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "not found"))
|
||||
(get-comments conn thread-id)))))
|
||||
|
||||
(def sql:get-comments
|
||||
|
||||
@@ -7,9 +7,10 @@
|
||||
(ns app.rpc.commands.demo
|
||||
"A demo specific mutations."
|
||||
(:require
|
||||
[app.auth :refer [derive-password]]
|
||||
[app.auth :refer [derive-password-weak]]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.time :as ct]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.loggers.audit :as audit]
|
||||
@@ -17,25 +18,33 @@
|
||||
[app.rpc.commands.auth :as auth]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.util.services :as sv]
|
||||
[app.worker :as wrk]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.nonce :as bn]))
|
||||
|
||||
(def ^:private
|
||||
schema:create-demo-profile
|
||||
[:map
|
||||
[:skip-onboarding {:optional true} ::sm/boolean]])
|
||||
|
||||
(sv/defmethod ::create-demo-profile
|
||||
"A command that is responsible of creating a demo purpose
|
||||
profile. It only works if the `demo-users` flag is enabled in the
|
||||
configuration."
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::doc/changes ["1.15" "This method is migrated from mutations to commands."]}
|
||||
[cfg _]
|
||||
::doc/changes [["1.15" "This method is migrated from mutations to commands."]
|
||||
["2.18" "Add optional `skip-onboarding` param. When true, the profile is created with `onboarding-viewed` and `release-notes-viewed` (current version) set, skipping the onboarding flow."]]
|
||||
::sm/params schema:create-demo-profile}
|
||||
[cfg {:keys [skip-onboarding]}]
|
||||
|
||||
(when-not (contains? cf/flags :demo-users)
|
||||
(ex/raise :type :validation
|
||||
:code :demo-users-not-allowed
|
||||
:hint "Demo users are disabled by config."))
|
||||
|
||||
(let [sem (System/currentTimeMillis)
|
||||
email (str "demo-" sem ".demo@example.com")
|
||||
(let [sem (uuid/next)
|
||||
email (str "demo-" sem "@demo.example.com")
|
||||
fullname (str "Demo User " sem)
|
||||
|
||||
password (-> (bn/random-bytes 16)
|
||||
@@ -46,13 +55,23 @@
|
||||
:fullname fullname
|
||||
:is-active true
|
||||
:is-demo true
|
||||
:deleted-at (ct/in-future (cf/get-deletion-delay))
|
||||
:password (derive-password password)
|
||||
:props {}}
|
||||
:password (derive-password-weak password)
|
||||
:props (cond-> {}
|
||||
skip-onboarding (assoc :onboarding-viewed true
|
||||
;; Redundant today: auth/create-profile
|
||||
;; overwrites this with the current
|
||||
;; version, kept so the skip does not
|
||||
;; depend on that default.
|
||||
:release-notes-viewed (:main cf/version)))}
|
||||
profile (db/tx-run! cfg (fn [cfg]
|
||||
(->> (auth/create-profile cfg params)
|
||||
(auth/create-profile-rels cfg))))]
|
||||
|
||||
(wrk/submit! (-> cfg
|
||||
(assoc ::wrk/task :demo-purge)
|
||||
(assoc ::wrk/delay (cf/get-deletion-delay))
|
||||
(assoc ::wrk/params {:profile-id (:id profile)})))
|
||||
|
||||
(with-meta {:email email
|
||||
:password password}
|
||||
{::audit/profile-id (:id profile)})))
|
||||
|
||||
@@ -95,18 +95,23 @@
|
||||
(def check-read-permissions!
|
||||
(perms/make-check-fn has-read-permissions?))
|
||||
|
||||
;; A user has comment permissions if she has read permissions, or
|
||||
;; explicit comment permissions through the share-id
|
||||
;; A user has comment permissions if:
|
||||
;; - For :membership type: they have read permissions OR explicit comment permissions
|
||||
;; - For :share-link type: they must have explicit comment permissions (who-comment=all)
|
||||
;; This prevents share-link holders with who-comment=team from bypassing the restriction
|
||||
|
||||
(defn check-comment-permissions!
|
||||
[cfg profile-id file-id share-id]
|
||||
(let [perms (perms/get-file-read-permissions cfg profile-id file-id share-id)
|
||||
can-read (has-read-permissions? perms)
|
||||
can-comment (has-comment-permissions? perms)]
|
||||
(when-not (or can-read can-comment)
|
||||
(let [perms (perms/get-file-read-permissions cfg profile-id file-id share-id)
|
||||
allowed? (if (= :share-link (:type perms))
|
||||
(has-comment-permissions? perms)
|
||||
(or (has-read-permissions? perms)
|
||||
(has-comment-permissions? perms)))]
|
||||
(when-not allowed?
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "not found"))))
|
||||
:hint "not found"))
|
||||
perms))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; QUERY COMMANDS
|
||||
@@ -236,6 +241,18 @@
|
||||
(some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"})
|
||||
(update :data blob/decode)))
|
||||
|
||||
(defn- check-fragment-scope!
|
||||
"Checks that the fragment is reachable from the pages authorized by
|
||||
the share-link. Raises a :not-found exception if the fragment is not reachable."
|
||||
[cfg file-id fragment-id pages]
|
||||
(let [fdata (-> (bfc/get-file cfg file-id :read-only? true)
|
||||
(get :data)
|
||||
(update :pages-index select-keys pages))]
|
||||
(when-not (contains? (feat.fdata/get-used-pointer-ids fdata) fragment-id)
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "object not found"))))
|
||||
|
||||
(sv/defmethod ::get-file-fragment
|
||||
"Retrieve a file fragment by its ID. Only authenticated users."
|
||||
{::doc/added "1.17"
|
||||
@@ -245,11 +262,9 @@
|
||||
[cfg {:keys [::rpc/profile-id file-id fragment-id share-id]}]
|
||||
(db/run! cfg (fn [cfg]
|
||||
(let [perms (perms/get-file-read-permissions cfg profile-id file-id share-id)]
|
||||
(when (= :share-link (:type perms))
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "object not found"))
|
||||
(check-read-permissions! perms)
|
||||
(when (= :share-link (:type perms))
|
||||
(check-fragment-scope! cfg file-id fragment-id (:pages perms)))
|
||||
(-> (get-file-fragment cfg file-id fragment-id)
|
||||
(rph/with-http-cache long-cache-duration))))))
|
||||
|
||||
@@ -515,8 +530,11 @@
|
||||
(def ^:private file-summary-cache-key-ttl
|
||||
(ct/duration {:days 30}))
|
||||
|
||||
(def file-summary-cache-key-prefix
|
||||
"penpot.library-summary.")
|
||||
(defn file-summary-cache-key
|
||||
"Build the redis cache key for the file library summary. The tenant is
|
||||
included to prevent key collisions between tenants sharing a redis instance"
|
||||
[id]
|
||||
(str "penpot.library-summary." (cf/get :tenant) "." id))
|
||||
|
||||
(defn- get-file-with-summary
|
||||
"Get a file without data with a summary of its local library content"
|
||||
@@ -545,7 +563,7 @@
|
||||
(rds/build-set-args {:ex file-summary-cache-key-ttl})))]
|
||||
|
||||
(if (contains? cf/flags :redis-cache)
|
||||
(let [cache-key (str file-summary-cache-key-prefix id)]
|
||||
(let [cache-key (file-summary-cache-key id)]
|
||||
(or (rds/run! cfg get-from-cache cache-key)
|
||||
(let [file (calculate-from-db)]
|
||||
(rds/run! cfg persist-to-cache (:library-summary file) cache-key)
|
||||
|
||||
@@ -322,7 +322,7 @@
|
||||
(defn- invalidate-caches!
|
||||
[cfg {:keys [id] :as file}]
|
||||
(rds/run! cfg (fn [{:keys [::rds/conn]}]
|
||||
(let [key (str files/file-summary-cache-key-prefix id)]
|
||||
(let [key (files/file-summary-cache-key id)]
|
||||
(rds/del conn key)))))
|
||||
|
||||
(defn- attach-snapshot
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
(ns app.rpc.commands.ldap
|
||||
(:require
|
||||
[app.auth.ldap :as ldap]
|
||||
[app.auth.login-lockout :as login-lockout]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.db :as db]
|
||||
@@ -39,62 +38,42 @@
|
||||
::doc/added "1.15"
|
||||
::doc/module :auth
|
||||
::sm/params schema:login-with-ldap}
|
||||
[{:keys [::ldap/provider ::db/pool] :as cfg} params]
|
||||
[{:keys [::ldap/provider] :as cfg} params]
|
||||
(when-not provider
|
||||
(ex/raise :type :restriction
|
||||
:code :ldap-not-initialized
|
||||
:hint "ldap auth provider is not initialized"))
|
||||
|
||||
(let [email (profile/clean-email (:email params))
|
||||
profile (profile/get-profile-by-email pool email)]
|
||||
(let [info (ldap/authenticate provider params)]
|
||||
(when-not info
|
||||
(ex/raise :type :validation
|
||||
:code :wrong-credentials))
|
||||
|
||||
(when profile
|
||||
(let [result (login-lockout/locked? cfg (:id profile))]
|
||||
(when (:locked? result)
|
||||
(ex/raise :type :rate-limit
|
||||
:code :account-locked
|
||||
:hint "account locked due to too many failed login attempts"
|
||||
:ttl (:ttl result)))))
|
||||
(let [profile (login-or-register cfg info)]
|
||||
|
||||
(let [info (ldap/authenticate provider params)]
|
||||
(when-not info
|
||||
(let [result (when profile
|
||||
(login-lockout/record-failed-attempt! cfg (:id profile)))]
|
||||
(if (and result (:locked? result))
|
||||
(ex/raise :type :rate-limit
|
||||
:code :account-locked
|
||||
:hint "account locked due to too many failed login attempts"
|
||||
:ttl (:ttl result))
|
||||
(ex/raise :type :validation
|
||||
:code :wrong-credentials))))
|
||||
(when (:is-blocked profile)
|
||||
(ex/raise :type :restriction
|
||||
:code :profile-blocked))
|
||||
|
||||
(let [profile (or profile (login-or-register cfg info))]
|
||||
|
||||
(when (:is-blocked profile)
|
||||
(ex/raise :type :restriction
|
||||
:code :profile-blocked))
|
||||
|
||||
(login-lockout/clear-attempts! cfg (:id profile))
|
||||
|
||||
(if-let [token (:invitation-token params)]
|
||||
;; If invitation token comes in params, this is because the
|
||||
;; user comes from team-invitation process; in this case,
|
||||
;; regenerate token and send back to the user a new invitation
|
||||
;; token (and mark current session as logged).
|
||||
(let [claims (tokens/verify cfg {:token token :iss :team-invitation})
|
||||
claims (assoc claims
|
||||
:member-id (:id profile)
|
||||
:member-email (:email profile))
|
||||
token (tokens/generate cfg claims)]
|
||||
(-> {:invitation-token token}
|
||||
(rph/with-transform (session/create-fn cfg profile))
|
||||
(rph/with-meta {::audit/props (:props profile)
|
||||
::audit/profile-id (:id profile)})))
|
||||
|
||||
(-> (profile/strip-private-attrs profile)
|
||||
(if-let [token (:invitation-token params)]
|
||||
;; If invitation token comes in params, this is because the
|
||||
;; user comes from team-invitation process; in this case,
|
||||
;; regenerate token and send back to the user a new invitation
|
||||
;; token (and mark current session as logged).
|
||||
(let [claims (tokens/verify cfg {:token token :iss :team-invitation})
|
||||
claims (assoc claims
|
||||
:member-id (:id profile)
|
||||
:member-email (:email profile))
|
||||
token (tokens/generate cfg claims)]
|
||||
(-> {:invitation-token token}
|
||||
(rph/with-transform (session/create-fn cfg profile))
|
||||
(rph/with-meta {::audit/props (:props profile)
|
||||
::audit/profile-id (:id profile)})))))))
|
||||
::audit/profile-id (:id profile)})))
|
||||
|
||||
(-> (profile/strip-private-attrs profile)
|
||||
(rph/with-transform (session/create-fn cfg profile))
|
||||
(rph/with-meta {::audit/props (:props profile)
|
||||
::audit/profile-id (:id profile)}))))))
|
||||
|
||||
(defn- login-or-register
|
||||
[cfg info]
|
||||
|
||||
@@ -425,8 +425,11 @@
|
||||
cfg (-> cfg
|
||||
(assoc ::bfc/project-id project-id)
|
||||
(assoc ::bfc/profile-id profile-id)
|
||||
(assoc ::bfc/team-id (:id team))
|
||||
(assoc ::bfc/input template)
|
||||
(assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team)))
|
||||
(assoc ::bfc/features (cfeat/get-team-enabled-features cf/flags team))
|
||||
(assoc ::bfc/import-max-object-size (cf/get :binfile-import-max-object-size))
|
||||
(assoc ::bfc/import-max-zip-entries (cf/get :binfile-import-max-zip-entries)))
|
||||
|
||||
result (if (= format :binfile-v3)
|
||||
(bf.v3/import-files! cfg)
|
||||
|
||||
@@ -336,7 +336,7 @@
|
||||
(doseq [{:keys [id reassign-to]} teams-to-leave]
|
||||
(teams/leave-team cfg {:profile-id profile-id :id id :reassign-to reassign-to}))
|
||||
|
||||
;; Process organization "Your Penpot" team: keep with prefix if needed, otherwise delete.
|
||||
;; Process organization "Personal Projects" team: keep with prefix if needed, otherwise delete.
|
||||
(when default-team-id
|
||||
(if keep-default-team?
|
||||
(db/exec! conn [sql:prefix-team-name-and-unset-default organization-prefix default-team-id])
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.rpc.commands.plugins
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.types.plugins :as ctp]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.util.services :as sv]))
|
||||
|
||||
(defn- validate-plugin-permissions!
|
||||
"Validates that all permissions in the plugin are within the valid set."
|
||||
[plugin]
|
||||
(let [permissions (:permissions plugin)
|
||||
invalid (remove ctp/valid-permissions permissions)]
|
||||
(when (seq invalid)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-plugin-permissions
|
||||
:hint (str "Invalid permissions: " (pr-str (set invalid)))
|
||||
:invalid-permissions (set invalid)))))
|
||||
|
||||
(def ^:private
|
||||
schema:add-profile-plugin
|
||||
[:map {:title "add-profile-plugin"}
|
||||
[:plugin ctp/schema:registry-entry]])
|
||||
|
||||
(sv/defmethod ::add-profile-plugin
|
||||
{::doc/added "2.18"
|
||||
::sm/params schema:add-profile-plugin
|
||||
::sm/result ctp/schema:registry-entry
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id plugin]}]
|
||||
(validate-plugin-permissions! plugin)
|
||||
|
||||
(let [profile (profile/get-profile conn profile-id ::db/for-update true)
|
||||
plugins (get-in profile [:props :plugins] {:ids [] :data {}})
|
||||
plugin-id (:plugin-id plugin)
|
||||
plugins (-> plugins
|
||||
(update :ids #(vec (distinct (conj % plugin-id))))
|
||||
(assoc-in [:data plugin-id] plugin))]
|
||||
(db/update! conn :profile
|
||||
{:props (db/tjson (assoc (:props profile) :plugins plugins))}
|
||||
{:id profile-id}
|
||||
{::db/return-keys false})
|
||||
plugin))
|
||||
|
||||
(def ^:private
|
||||
schema:remove-profile-plugin
|
||||
[:map {:title "remove-profile-plugin"}
|
||||
[:plugin-id ::sm/uuid]])
|
||||
|
||||
(sv/defmethod ::remove-profile-plugin
|
||||
{::doc/added "2.18"
|
||||
::sm/params schema:remove-profile-plugin
|
||||
::sm/result :nil
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id plugin-id]}]
|
||||
(let [profile (profile/get-profile conn profile-id ::db/for-update true)
|
||||
plugins (get-in profile [:props :plugins] {:ids [] :data {}})
|
||||
plugin-id-str (str plugin-id)
|
||||
plugins (-> plugins
|
||||
(update :ids #(vec (remove (partial = plugin-id-str) %)))
|
||||
(update :data dissoc plugin-id-str))]
|
||||
(db/update! conn :profile
|
||||
{:props (db/tjson (assoc (:props profile) :plugins plugins))}
|
||||
{:id profile-id}
|
||||
{::db/return-keys false})
|
||||
nil))
|
||||
@@ -12,7 +12,7 @@
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.plugins :refer [schema:plugin-registry]]
|
||||
[app.common.types.plugins :as ctp]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
@@ -54,11 +54,11 @@
|
||||
|
||||
(def system-managed-props
|
||||
"Props keys managed by the system (not user-writable via RPC)."
|
||||
#{:subscription})
|
||||
#{:subscription :plugins})
|
||||
|
||||
(def schema:props
|
||||
[:map {:title "ProfileProps" :closed true}
|
||||
[:plugins {:optional true} schema:plugin-registry]
|
||||
[:plugins {:optional true} ctp/schema:plugin-registry]
|
||||
[:renderer {:optional true} [::sm/one-of #{:svg :wasm}]]
|
||||
[:mcp-enabled {:optional true} ::sm/boolean]
|
||||
[:newsletter-updates {:optional true} ::sm/boolean]
|
||||
@@ -78,6 +78,10 @@
|
||||
[:map-of {:gen/max 10} :keyword [:map-of :keyword :string]]]
|
||||
[:nudge {:optional true} schema:nudge]])
|
||||
|
||||
(def schema:props-writeable
|
||||
"Props schema for user-writable fields (excludes system-managed keys)."
|
||||
(reduce sm/dissoc-key schema:props system-managed-props))
|
||||
|
||||
(def schema:profile
|
||||
[:map {:title "Profile"}
|
||||
[:id ::sm/uuid]
|
||||
@@ -141,9 +145,7 @@
|
||||
(defn get-profile
|
||||
"Get profile by id. Throws not-found exception if no profile found."
|
||||
[conn id & {:as opts}]
|
||||
;; NOTE: We need to set ::db/remove-deleted to false because demo profiles
|
||||
;; are created with a set deleted-at value
|
||||
(-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false))
|
||||
(-> (db/get-by-id conn :profile id opts)
|
||||
(decode-row)))
|
||||
|
||||
;; --- MUTATION: Update Profile (own)
|
||||
@@ -465,7 +467,7 @@
|
||||
(def ^:private
|
||||
schema:update-profile-props
|
||||
[:map {:title "update-profile-props"}
|
||||
[:props schema:props]])
|
||||
[:props schema:props-writeable]])
|
||||
|
||||
(defn update-profile-props
|
||||
[{:keys [::db/conn] :as cfg} profile-id props]
|
||||
@@ -524,7 +526,7 @@
|
||||
;; Penpot back through two paths: ::notify-user-organizations-deletion
|
||||
;; (during delete-owned-organizations) and ::notify-organization-deletion.
|
||||
;; Both preserve organization teams unchanged and only prefix or delete
|
||||
;; imported "Your Penpot" teams according to whether they still have files.
|
||||
;; imported "Personal Projects" teams according to whether they still have files.
|
||||
;; Let Nitrate clean up the data associated with the deleted Penpot user:
|
||||
;; owned organizations, remaining memberships, and subscription cancellation.
|
||||
(when (contains? cf/flags :admin-console)
|
||||
|
||||
@@ -575,7 +575,7 @@
|
||||
(set/difference cfeat/frontend-only-features)
|
||||
(set/difference cfeat/no-team-inheritable-features))
|
||||
params {:profile-id profile-id
|
||||
:name "Your Penpot"
|
||||
:name "Personal Projects"
|
||||
:features features
|
||||
:organization-id organization-id
|
||||
:is-default true}
|
||||
@@ -829,7 +829,7 @@
|
||||
:code :only-owner-can-delete-team))
|
||||
|
||||
;; Protect the user's personal default team from deletion.
|
||||
;; Organization-scoped default teams ("Your Penpot") are allowed to be deleted when they have no files.
|
||||
;; Organization-scoped default teams ("Personal Projects") are allowed to be deleted when they have no files.
|
||||
(when (and (:is-default team) (not in-organization?))
|
||||
(ex/raise :type :validation
|
||||
:code :non-deletable-team
|
||||
@@ -944,8 +944,10 @@
|
||||
::sm/params schema:delete-team-member
|
||||
::db/transaction true}
|
||||
[{:keys [::db/conn ::mbus/msgbus] :as cfg} {:keys [::rpc/profile-id team-id member-id] :as params}]
|
||||
(let [team (get-team conn :profile-id profile-id :team-id team-id)
|
||||
perms (get-permissions conn profile-id team-id)]
|
||||
(let [team (get-team conn :profile-id profile-id :team-id team-id)
|
||||
perms (get-permissions conn profile-id team-id)
|
||||
members (get-team-members conn team-id)
|
||||
member (d/seek #(= member-id (:id %)) members)]
|
||||
(when-not (or (:is-owner perms)
|
||||
(:is-admin perms))
|
||||
(ex/raise :type :validation
|
||||
@@ -955,6 +957,15 @@
|
||||
(ex/raise :type :validation
|
||||
:code :cant-remove-yourself))
|
||||
|
||||
(when-not member
|
||||
(ex/raise :type :not-found
|
||||
:code :member-does-not-exist))
|
||||
|
||||
(when (and (:is-owner member)
|
||||
(not (:is-owner perms)))
|
||||
(ex/raise :type :validation
|
||||
:code :cant-remove-owner))
|
||||
|
||||
(db/delete! conn :team-profile-rel {:profile-id member-id
|
||||
:team-id team-id})
|
||||
|
||||
|
||||
@@ -235,9 +235,8 @@
|
||||
:organization-name (:name organization)
|
||||
:member-email (:email-to invitation)
|
||||
:member-id (:id member)
|
||||
:role role}
|
||||
organization
|
||||
(assoc :user-who-send-invitation (str profile-id))
|
||||
:role role
|
||||
:user-who-send-invitation (str profile-id)}
|
||||
|
||||
(not organization)
|
||||
(assoc :team-belongs-to-organization (boolean team-organization-id)
|
||||
|
||||
@@ -308,7 +308,9 @@
|
||||
(assoc :name "accept-organization-invitation")
|
||||
(assoc :props
|
||||
(-> props
|
||||
(assoc :organization-id organization-id-on-add)
|
||||
(assoc :organization-id organization-id-on-add
|
||||
:user-id (:id profile)
|
||||
:user-who-send-invitation (:created-by invitation))
|
||||
(audit/clean-props))))))
|
||||
|
||||
(cond-> (assoc claims :state :created)
|
||||
@@ -325,6 +327,8 @@
|
||||
(assoc :organization-id organization-id-on-add
|
||||
:organization-member-add-source organization-add-source
|
||||
:belongs-to-team-on-add (boolean team-id)
|
||||
:user-id (:id profile)
|
||||
:user-who-send-invitation (:created-by invitation)
|
||||
:organization-member-count-before
|
||||
organization-member-count-before)
|
||||
(audit/clean-props))}))))))
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
(assoc :can-read true)))
|
||||
|
||||
(defn- get-view-only-bundle
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file-id share-id ::perms] :as params}]
|
||||
(let [file (bfc/get-file cfg file-id)
|
||||
|
||||
project (db/get conn :project
|
||||
@@ -89,16 +89,18 @@
|
||||
(mapv (fn [{:keys [id] :as lib}]
|
||||
(merge lib (bfc/get-file cfg id)))))
|
||||
|
||||
links (->> (db/query conn :share-link {:file-id file-id})
|
||||
(mapv (fn [row]
|
||||
(-> row
|
||||
(update :pages db/decode-pgarray #{})
|
||||
;; NOTE: the flags are deprecated but are still present
|
||||
;; on the table on old rows. The flags are pgarray and
|
||||
;; for avoid decoding it (because they are no longer used
|
||||
;; on frontend) we just dissoc the column attribute from
|
||||
;; row.
|
||||
(dissoc :flags)))))
|
||||
links (cond->> (->> (db/query conn :share-link {:file-id file-id})
|
||||
(mapv (fn [row]
|
||||
(-> row
|
||||
(update :pages db/decode-pgarray #{})
|
||||
;; NOTE: the flags are deprecated but are still present
|
||||
;; on the table on old rows. The flags are pgarray and
|
||||
;; for avoid decoding it (because they are no longer used
|
||||
;; on frontend) we just dissoc the column attribute from
|
||||
;; row.
|
||||
(dissoc :flags)))))
|
||||
(= :share-link (:type perms))
|
||||
(filterv #(= (:id %) share-id)))
|
||||
|
||||
fonts (db/query conn :team-font-variant
|
||||
{:team-id (:id team)
|
||||
|
||||
@@ -318,7 +318,7 @@ RETURNING id, deleted_at;")
|
||||
|
||||
(defn manage-deleted-organization-teams
|
||||
"For a deleted organization, preserve organization teams unchanged and only prefix or
|
||||
delete member Your Penpot teams depending on whether they still contain files."
|
||||
delete member Personal Projects teams depending on whether they still contain files."
|
||||
[cfg {:keys [organization-id organization-name teams]}]
|
||||
(let [all-team-ids (->> teams
|
||||
(map :id)
|
||||
@@ -347,13 +347,13 @@ RETURNING id, deleted_at;")
|
||||
teams-to-delete (->> your-penpot-team-ids (remove teams-with-files) (into []))]
|
||||
|
||||
;; Organization teams move to the fallback organization unchanged. Only imported
|
||||
;; Your Penpot teams keep the organization prefix when they still have files.
|
||||
;; Personal Projects teams keep the organization prefix when they still have files.
|
||||
(when (seq teams-to-prefix)
|
||||
(db/exec! conn [sql:prefix-teams-name-and-unset-default
|
||||
organization-prefix
|
||||
(db/create-array conn "uuid" teams-to-prefix)]))
|
||||
|
||||
;; Empty imported Your Penpot teams disappear entirely.
|
||||
;; Empty imported Personal Projects teams disappear entirely.
|
||||
(soft-delete-teams! cfg teams-to-delete)
|
||||
|
||||
(notifications/notify-organization-deletion cfg organization-id organization-name all-team-ids teams-to-delete)
|
||||
@@ -362,7 +362,7 @@ RETURNING id, deleted_at;")
|
||||
|
||||
(sv/defmethod ::notify-organization-deletion
|
||||
"For a deleted organization, preserve organization teams and only prefix or delete
|
||||
imported Your Penpot teams before notifying connected users."
|
||||
imported Personal Projects before notifying connected users."
|
||||
{::doc/added "2.18"
|
||||
::sm/params schema:notify-organization-deletion
|
||||
::rpc/auth false}
|
||||
@@ -382,7 +382,7 @@ RETURNING id, deleted_at;")
|
||||
|
||||
(sv/defmethod ::notify-user-organizations-deletion
|
||||
"For a given user, find all owned organizations and apply the deleted-organization
|
||||
transfer rules to their imported Your Penpot teams."
|
||||
transfer rules to their imported Personal Projects teams."
|
||||
{::doc/added "2.18"
|
||||
::sm/params schema:notify-user-organizations-deletion
|
||||
::nitrate/sso false}
|
||||
@@ -1045,3 +1045,18 @@ RETURNING id, deleted_at;")
|
||||
(update acc :created conj email)))))
|
||||
{:created [] :skipped []}
|
||||
emails)))))
|
||||
|
||||
;; ---- API: get-air-gapped
|
||||
|
||||
(def ^:private schema:get-air-gapped-result
|
||||
[:map
|
||||
[:air-gapped ::sm/boolean]])
|
||||
|
||||
(sv/defmethod ::get-air-gapped
|
||||
"Returns whether this Penpot instance runs in air-gapped mode."
|
||||
{::doc/added "2.18"
|
||||
::sm/params [:map]
|
||||
::sm/result schema:get-air-gapped-result
|
||||
::rpc/auth false}
|
||||
[_cfg _params]
|
||||
{:air-gapped (contains? cf/flags :air-gapped-conf)})
|
||||
@@ -232,7 +232,7 @@
|
||||
[:enum
|
||||
"customer_service"
|
||||
"low_quality"
|
||||
"missing_feature"
|
||||
"missing_features"
|
||||
"other"
|
||||
"switched_service"
|
||||
"too_complex"
|
||||
|
||||
@@ -398,6 +398,53 @@
|
||||
(println (sm/humanize-explain explain))
|
||||
(ex/print-throwable cause))))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; GRAPH / LADYBUG
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; The graph namespaces resolve at call time, never at the top of this
|
||||
;; namespace. `app.graph.ladybug` imports `com.ladybugdb.*`, and this namespace
|
||||
;; loads with the REPL server on every boot, so a top-level require would link
|
||||
;; the Ladybug native library into every backend, graph or not. Calling one of
|
||||
;; the functions below loads the library at that point: the operator has asked
|
||||
;; for it explicitly. The `:graph` flag gates the request path
|
||||
;; (`app.http.debug`), not the REPL.
|
||||
|
||||
(defn graph-smoke-test!
|
||||
"Execute a basic Ladybug smoke test (CREATE + count).
|
||||
|
||||
Uses the embedded Ladybug Java API. Use :db-path \":memory:\" (default)
|
||||
or a filesystem path such as /tmp/test.lbug."
|
||||
[& {:keys [db-path] :or {db-path ":memory:"}}]
|
||||
((requiring-resolve 'app.graph.ladybug/smoke-test!) :db-path db-path))
|
||||
|
||||
(defn graph-query-test!
|
||||
"Query Document count for a file's graph db (REPL diagnostic)."
|
||||
[file-id & {:keys [db-path]}]
|
||||
(let [file-id (h/parse-uuid file-id)
|
||||
db-path (or db-path ((requiring-resolve 'app.graph.ladybug/db-path-for-file) file-id))
|
||||
query-scalar! (requiring-resolve 'app.graph.ladybug/query-scalar!)
|
||||
stmt "MATCH (n:Document) RETURN count(n) AS Document_c;"]
|
||||
(query-scalar! db-path stmt)))
|
||||
|
||||
(defn ingest-file-to-graph!
|
||||
"Project a Penpot file into a per-file Ladybug database.
|
||||
|
||||
Loads and realizes the file from the database, ensures the slice schema,
|
||||
projects Document/Page/shape nodes, and returns graph stats.
|
||||
|
||||
Options:
|
||||
- `:db-path` path or `:memory:`
|
||||
- `:reset-db?` delete any existing db first (default true)
|
||||
- `:skip-stats?` skip post-ingest MATCH count queries (default false)"
|
||||
[file-id & opts]
|
||||
(let [ingest-file! (requiring-resolve 'app.graph.ingest/ingest-file!)
|
||||
print-ingest! (requiring-resolve 'app.graph.report/print-ingest!)
|
||||
result (apply ingest-file! sys/system file-id opts)]
|
||||
(print-ingest! result)
|
||||
result))
|
||||
|
||||
|
||||
(defn repair-file!
|
||||
"Repair the list of errors detected by validation."
|
||||
[file-id & {:keys [rollback?] :or {rollback? true} :as options}]
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.tasks.demo-purge
|
||||
"Task handler for delayed demo profile deletion. Submitted at demo
|
||||
creation time with a delay matching the configured deletion-delay."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.db :as db]
|
||||
[app.worker :as wrk]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(defmethod ig/assert-key ::handler
|
||||
[_ params]
|
||||
(assert (db/pool? (::db/pool params)) "expected a valid database pool"))
|
||||
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [{:keys [props]}]
|
||||
(let [profile-id (get props :profile-id)
|
||||
now (ct/now)]
|
||||
|
||||
(l/trc :hint "demo-purge" :profile-id (str profile-id))
|
||||
|
||||
;; Mark the profile for immediate deletion
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(db/update! conn :profile
|
||||
{:deleted-at now}
|
||||
{:id profile-id}
|
||||
{::db/return-keys false})
|
||||
(wrk/submit!
|
||||
(-> cfg
|
||||
(assoc ::wrk/task :delete-object)
|
||||
(assoc ::wrk/params {:object :profile
|
||||
:deleted-at now
|
||||
:id profile-id}))))))))
|
||||
@@ -13,6 +13,7 @@
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as fdata]
|
||||
[app.storage :as sto]
|
||||
[app.tasks.delete-object :as dobj]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(def ^:private sql:get-profiles
|
||||
@@ -33,6 +34,11 @@
|
||||
;; Mark as deleted the storage object
|
||||
(some->> photo-id (sto/touch-object! storage))
|
||||
|
||||
;; Cascade soft-delete to owned teams, projects, files, etc.
|
||||
(dobj/delete-object cfg {:object :profile
|
||||
:id id
|
||||
:deleted-at timestamp})
|
||||
|
||||
(let [affected (-> (db/delete! conn :profile {:id id})
|
||||
(db/get-update-count))]
|
||||
(+ total affected)))
|
||||
|
||||
@@ -5,7 +5,12 @@
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.util.ssrf
|
||||
"URL/host validation to prevent Server-Side Request Forgery."
|
||||
"URL/host validation to prevent Server-Side Request Forgery.
|
||||
|
||||
The blocklist covers the standard JVM InetAddress classifications plus
|
||||
explicit ranges: IPv6 ULA, IPv4-mapped loopback, cloud metadata,
|
||||
operator-supplied CIDRs and the IPv6 transition mechanisms NAT64, 6to4
|
||||
and Teredo."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
@@ -122,6 +127,20 @@
|
||||
;; Check the embedded IPv4 is loopback (127.x.x.x)
|
||||
(= (bit-and (aget bs 12) 0xFF) 127))))
|
||||
|
||||
(defn- transition-prefix
|
||||
"Classify a 16-byte IPv6 address into its transition mechanism:
|
||||
:nat64 (64:ff9b::/96), :6to4 (2002::/16), :teredo (2001:0000::/32) or nil."
|
||||
[^bytes bs]
|
||||
(let [b0 (bit-and (aget bs 0) 0xFF)
|
||||
b1 (bit-and (aget bs 1) 0xFF)
|
||||
b2 (bit-and (aget bs 2) 0xFF)
|
||||
b3 (bit-and (aget bs 3) 0xFF)]
|
||||
(cond
|
||||
(and (= b0 0x00) (= b1 0x64) (= b2 0xFF) (= b3 0x9B)) :nat64
|
||||
(and (= b0 0x20) (= b1 0x02)) :6to4
|
||||
(and (= b0 0x20) (= b1 0x01) (= b2 0x00) (= b3 0x00)) :teredo
|
||||
:else nil)))
|
||||
|
||||
(defn- blocked-address?
|
||||
"Check if an InetAddress should be blocked. Returns true if blocked."
|
||||
[^InetAddress addr]
|
||||
@@ -141,12 +160,15 @@
|
||||
;; Cloud metadata IPs (exact match)
|
||||
(contains? cloud-metadata-ips (.getHostAddress addr))
|
||||
|
||||
;; Extra blocked CIDRs (IPv4 only)
|
||||
;; Extra blocked CIDRs (IPv4 only) and IPv6 transition mechanisms
|
||||
(let [bs (.getAddress addr)]
|
||||
(if (= (alength bs) 4)
|
||||
(or (some #(in-cidr4? bs %) extra-blocked-ranges)
|
||||
(some #(in-cidr4? bs %) extra-blocked-cidrs))
|
||||
false))))
|
||||
;; IPv6 transition mechanisms (NAT64/6to4/Teredo): the range is
|
||||
;; rejected outright.
|
||||
(boolean (when (= (alength bs) 16)
|
||||
(transition-prefix bs)))))))
|
||||
|
||||
(defn resolve-host
|
||||
"Resolve a hostname to all InetAddress objects. Wraps InetAddress/getAllByName
|
||||
@@ -163,8 +185,10 @@
|
||||
- host must resolve to at least one address, and
|
||||
- **every** resolved address must NOT be in the blocklist
|
||||
(loopback, link-local, site-local, multicast, any-local,
|
||||
cloud-metadata 169.254.169.254, IPv6 ULA fc00::/7, IPv4-mapped
|
||||
IPv6 of any blocked IPv4, plus operator-supplied CIDRs).
|
||||
cloud-metadata 169.254.169.254, IPv6 ULA fc00::/7, IPv6 transition
|
||||
mechanisms NAT64 64:ff9b::/96, 6to4 2002::/16 and Teredo
|
||||
2001:0000::/32, IPv4-mapped IPv6 of any blocked IPv4,
|
||||
plus operator-supplied CIDRs).
|
||||
When the host is an IP literal (decimal/octal/hex/IPv6) it is
|
||||
normalized via `com.google.common.net.InetAddresses` before the
|
||||
check.
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.auth-login-lockout-test
|
||||
(:require
|
||||
[app.auth.login-lockout :as lol]
|
||||
[app.common.flags :as flags]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(t/use-fixtures :once (partial th/init-system))
|
||||
(t/use-fixtures :each (partial th/init-config [:enable-account-lockout]))
|
||||
|
||||
(t/deftest record-failed-attempt-returns-nil-when-disabled
|
||||
(with-redefs [app.config/flags (flags/parse flags/default th/default-flags)]
|
||||
(let [profile-id (uuid/next)
|
||||
result (lol/record-failed-attempt! th/*system* profile-id)]
|
||||
(t/is (nil? result)))))
|
||||
|
||||
(t/deftest record-failed-attempt-increments-counter
|
||||
(let [profile-id (uuid/next)
|
||||
r1 (lol/record-failed-attempt! th/*system* profile-id)
|
||||
r2 (lol/record-failed-attempt! th/*system* profile-id)
|
||||
r3 (lol/record-failed-attempt! th/*system* profile-id)]
|
||||
(t/is (map? r1))
|
||||
(t/is (= 1 (:count r1)))
|
||||
(t/is (false? (:locked? r1)))
|
||||
(t/is (= 2 (:count r2)))
|
||||
(t/is (false? (:locked? r2)))
|
||||
(t/is (= 3 (:count r3)))
|
||||
(t/is (false? (:locked? r3)))))
|
||||
|
||||
(t/deftest record-failed-attempt-locks-after-threshold
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 4]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(let [result (lol/record-failed-attempt! th/*system* profile-id)]
|
||||
(t/is (= 5 (:count result)))
|
||||
(t/is (:locked? result)))))
|
||||
|
||||
(t/deftest locked?-returns-false-below-threshold
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 3]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(t/is (false? (:locked? (lol/locked? th/*system* profile-id))))))
|
||||
|
||||
(t/deftest locked?-returns-true-at-threshold
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(t/is (:locked? (lol/locked? th/*system* profile-id)))))
|
||||
|
||||
(t/deftest locked?-returns-false-when-disabled
|
||||
(with-redefs [app.config/flags (flags/parse flags/default th/default-flags)]
|
||||
(let [profile-id (uuid/next)]
|
||||
(t/is (false? (:locked? (lol/locked? th/*system* profile-id)))))))
|
||||
|
||||
(t/deftest clear-attempts-resets-counter
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(t/is (:locked? (lol/locked? th/*system* profile-id)))
|
||||
(lol/clear-attempts! th/*system* profile-id)
|
||||
(t/is (false? (:locked? (lol/locked? th/*system* profile-id))))))
|
||||
|
||||
(t/deftest clear-attempts-is-no-op-when-disabled
|
||||
(with-redefs [app.config/flags (flags/parse flags/default th/default-flags)]
|
||||
(let [profile-id (uuid/next)]
|
||||
(t/is (nil? (lol/clear-attempts! th/*system* profile-id))))))
|
||||
|
||||
(t/deftest locked?-returns-false-for-unknown-profile
|
||||
(let [profile-id (uuid/next)]
|
||||
(t/is (false? (:locked? (lol/locked? th/*system* profile-id))))))
|
||||
|
||||
(t/deftest locked?-returns-ttl-when-locked
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (:locked? result))
|
||||
(t/is (pos-int? (:ttl result))
|
||||
"ttl should be a positive integer when locked"))))
|
||||
|
||||
(t/deftest locked?-returns-no-ttl-when-not-locked
|
||||
(let [profile-id (uuid/next)]
|
||||
(dotimes [i 2]
|
||||
(lol/record-failed-attempt! th/*system* profile-id))
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (false? (:locked? result)))
|
||||
(t/is (nil? (:ttl result))
|
||||
"ttl should be nil when not locked"))))
|
||||
|
||||
(t/deftest locked?-expires-after-window
|
||||
(let [profile-id (uuid/next)
|
||||
start (ct/inst "2026-01-01T00:00:00Z")]
|
||||
;; Record 5 attempts at t=0 → locked
|
||||
(binding [ct/*clock* (ct/fixed-clock start)]
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! th/*system* profile-id)))
|
||||
;; At t=5min → still locked
|
||||
(binding [ct/*clock* (ct/fixed-clock (ct/plus start {:minutes 5}))]
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (:locked? result) "should still be locked at 5min")
|
||||
(t/is (pos-int? (:ttl result)))))
|
||||
;; At t=16min → expired, not locked
|
||||
(binding [ct/*clock* (ct/fixed-clock (ct/plus start {:minutes 16}))]
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (false? (:locked? result)) "should be unlocked after window expires")
|
||||
(t/is (nil? (:ttl result)))))))
|
||||
|
||||
(t/deftest record-failed-attempt-resets-window-on-expiry
|
||||
(let [profile-id (uuid/next)
|
||||
start (ct/inst "2026-01-01T00:00:00Z")]
|
||||
;; Record 2 attempts at t=0
|
||||
(binding [ct/*clock* (ct/fixed-clock start)]
|
||||
(dotimes [i 2]
|
||||
(lol/record-failed-attempt! th/*system* profile-id)))
|
||||
;; At t=20min (after 15min window), counter should reset
|
||||
(binding [ct/*clock* (ct/fixed-clock (ct/plus start {:minutes 20}))]
|
||||
(let [result (lol/record-failed-attempt! th/*system* profile-id)]
|
||||
(t/is (= 1 (:count result)) "count should reset to 1 after window expires")
|
||||
(t/is (false? (:locked? result)))))))
|
||||
|
||||
(t/deftest locked?-ttl-decreases-over-time
|
||||
(let [profile-id (uuid/next)
|
||||
start (ct/inst "2026-01-01T00:00:00Z")]
|
||||
;; Record 5 attempts at t=0 → locked
|
||||
(binding [ct/*clock* (ct/fixed-clock start)]
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! th/*system* profile-id)))
|
||||
;; At t=5min → ttl should be ~600s (10min remaining)
|
||||
(binding [ct/*clock* (ct/fixed-clock (ct/plus start {:minutes 5}))]
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (:locked? result))
|
||||
(t/is (<= 590 (:ttl result) 600) "ttl should be ~600s at 5min")))
|
||||
;; At t=14min → ttl should be ~60s
|
||||
(binding [ct/*clock* (ct/fixed-clock (ct/plus start {:minutes 14}))]
|
||||
(let [result (lol/locked? th/*system* profile-id)]
|
||||
(t/is (:locked? result))
|
||||
(t/is (<= 50 (:ttl result) 70) "ttl should be ~60s at 14min")))))
|
||||
@@ -27,6 +27,7 @@
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[backend-tests.helpers :as th]
|
||||
[backend-tests.storage-test :as stt]
|
||||
[clojure.test :as t]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
@@ -1883,3 +1884,86 @@
|
||||
(t/is (some? ext-libs)
|
||||
"type :link-later should produce external-libraries even with include-libraries=true")
|
||||
(t/is (pos? (count ext-libs))))))
|
||||
|
||||
(t/deftest import-rejects-too-many-zip-entries
|
||||
;; import must reject ZIP files exceeding max-zip-entries
|
||||
(let [profile (th/create-profile* 1)
|
||||
file (prepare-simple-file profile)
|
||||
output (tmp/tempfile :suffix ".zip")]
|
||||
|
||||
(v3/export-files!
|
||||
(-> th/*system*
|
||||
(assoc ::bfc/ids #{(:id file)})
|
||||
(assoc ::bfc/export-type :detach-libraries))
|
||||
(io/output-stream output))
|
||||
|
||||
;; Import with max-zip-entries=1 — the exported ZIP has more entries
|
||||
(let [cfg (-> th/*system*
|
||||
(assoc ::bfc/project-id (:default-project-id profile))
|
||||
(assoc ::bfc/profile-id (:id profile))
|
||||
(assoc ::bfc/input output)
|
||||
(assoc ::bfc/import-max-zip-entries 1))
|
||||
out (try
|
||||
(v3/import-files! cfg)
|
||||
:no-error
|
||||
(catch Throwable e
|
||||
(let [d (or (ex-data e) (some-> (ex-cause e) ex-data))]
|
||||
d)))]
|
||||
(t/is (= :validation (:type out)))
|
||||
(t/is (= :too-many-zip-entries (:code out))))))
|
||||
|
||||
(defn- prepare-file-with-media
|
||||
"Creates a file with a media object backed by a real storage object,
|
||||
so that v3 export produces objects/ entries."
|
||||
[profile]
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(stt/configure-storage-backend))
|
||||
|
||||
sobject (sto/put-object! storage {::sto/content (sto/content "media-bytes")
|
||||
:content-type "image/svg+xml"
|
||||
:bucket "file-media-object"})
|
||||
|
||||
file (th/create-file* 1 {:profile-id (:id profile)
|
||||
:project-id (:default-project-id profile)
|
||||
:is-shared false})
|
||||
|
||||
mobj (th/create-file-media-object* {:file-id (:id file)
|
||||
:is-local true
|
||||
:media-id (:id sobject)})]
|
||||
(update-file!
|
||||
:file-id (:id file)
|
||||
:profile-id (:id profile)
|
||||
:revn 0
|
||||
:vern 0
|
||||
:changes
|
||||
[{:type :add-media
|
||||
:object mobj}])
|
||||
|
||||
(dissoc file :data)))
|
||||
|
||||
(t/deftest import-rejects-oversized-object
|
||||
;; import must reject storage objects exceeding max-object-size
|
||||
(let [profile (th/create-profile* 1)
|
||||
file (prepare-file-with-media profile)
|
||||
output (tmp/tempfile :suffix ".zip")]
|
||||
|
||||
(v3/export-files!
|
||||
(-> th/*system*
|
||||
(assoc ::bfc/ids #{(:id file)})
|
||||
(assoc ::bfc/export-type :detach-libraries))
|
||||
(io/output-stream output))
|
||||
|
||||
;; Import with max-object-size=1 — the media object will exceed this
|
||||
(let [cfg (-> th/*system*
|
||||
(assoc ::bfc/project-id (:default-project-id profile))
|
||||
(assoc ::bfc/profile-id (:id profile))
|
||||
(assoc ::bfc/input output)
|
||||
(assoc ::bfc/import-max-object-size 1))
|
||||
out (try
|
||||
(v3/import-files! cfg)
|
||||
:no-error
|
||||
(catch Throwable e
|
||||
(let [d (or (ex-data e) (some-> (ex-cause e) ex-data))]
|
||||
d)))]
|
||||
(t/is (= :validation (:type out)))
|
||||
(t/is (= :max-file-size-reached (:code out))))))
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
(ns backend-tests.db-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t])
|
||||
@@ -41,3 +42,13 @@
|
||||
|
||||
(t/testing "maximum pool size is reasonable"
|
||||
(t/is (pos? (:maximum-pool-size stats))))))
|
||||
|
||||
(t/deftest uuid->hash-code-is-deterministic
|
||||
(t/is (= (db/uuid->hash-code uuid/zero)
|
||||
(db/uuid->hash-code uuid/zero))))
|
||||
|
||||
(t/deftest uuid->hash-code-returns-long
|
||||
(t/is (instance? Long (db/uuid->hash-code uuid/zero))))
|
||||
|
||||
(t/deftest uuid->hash-code-stable-for-zero-uuid
|
||||
(t/is (= 3659997967308761462 (db/uuid->hash-code uuid/zero))))
|
||||
@@ -0,0 +1,47 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.demo-test
|
||||
(:require
|
||||
[app.common.time :as ct]
|
||||
[app.db :as db]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.tasks.demo-purge :as demo-purge]
|
||||
[app.worker :as wrk]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
|
||||
(t/deftest demo-profile-created-without-deleted-at
|
||||
(let [profile (th/create-profile* 999 {:is-demo true})]
|
||||
(t/is (true? (:is-demo profile)))
|
||||
(t/is (nil? (:deleted-at profile)))
|
||||
(t/is (some? (:id profile)))))
|
||||
|
||||
(t/deftest get-profile-finds-demo-user-without-override
|
||||
(let [profile (th/create-profile* 998 {:is-demo true})
|
||||
found (db/run! th/*pool*
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(profile/get-profile conn (:id profile))))]
|
||||
(t/is (some? found))
|
||||
(t/is (= (:id profile) (:id found)))))
|
||||
|
||||
(t/deftest demo-purge-handler-submits-delete-object
|
||||
(let [profile (th/create-profile* 996 {:is-demo true})
|
||||
handler (ig/init-key :app.tasks.demo-purge/handler
|
||||
{::db/pool th/*pool*})
|
||||
submitted (atom nil)]
|
||||
(with-redefs [wrk/submit! (fn [& {:keys [::wrk/task ::wrk/params]}]
|
||||
(reset! submitted {:task task :params params}))]
|
||||
(handler {:props {:profile-id (:id profile)
|
||||
:deleted-at (ct/now)}}))
|
||||
(t/is (= :delete-object (:task @submitted)))
|
||||
(t/is (= :profile (:object (:params @submitted))))
|
||||
(t/is (= (:id profile) (:id (:params @submitted))))
|
||||
(t/is (some? (:deleted-at (:params @submitted))))))
|
||||
@@ -0,0 +1,165 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.graph-binder-gate-test
|
||||
"Binder gate for the incremental-sync statement templates.
|
||||
|
||||
Every template `app.graph.sync` emits is *prepared* — parsed and bound by
|
||||
the engine against the live DDL — and never executed. A parse or bind
|
||||
failure (a renamed column, a reserved-word label emitted unquoted, a dropped
|
||||
table) turns the gate red here, before the statement can reach a live
|
||||
session.
|
||||
|
||||
One instance per template is the gate; per-column type coverage belongs to
|
||||
beadpot's schema diff, not here. The templates are `defn-`, so they are
|
||||
reached through their vars."
|
||||
(:require
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema.nodes :as nodes]
|
||||
[app.graph.sync]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(def ^:private create-node-statement #'app.graph.sync/create-node-statement)
|
||||
(def ^:private delete-node-statement #'app.graph.sync/delete-node-statement)
|
||||
(def ^:private create-edge-statement #'app.graph.sync/create-edge-statement)
|
||||
(def ^:private delete-edge-statement #'app.graph.sync/delete-edge-statement)
|
||||
(def ^:private set-edge-position-statement #'app.graph.sync/set-edge-position-statement)
|
||||
(def ^:private create-instance-of-statement #'app.graph.sync/create-instance-of-statement)
|
||||
(def ^:private delete-instance-of-statement #'app.graph.sync/delete-instance-of-statement)
|
||||
(def ^:private set-node-attr-statement #'app.graph.sync/set-node-attr-statement)
|
||||
(def ^:private set-page-name-statement #'app.graph.sync/set-page-name-statement)
|
||||
(def ^:private remove-node-attr-statement #'app.graph.sync/remove-node-attr-statement)
|
||||
(def ^:private set-document-revision-statement #'app.graph.sync/set-document-revision-statement)
|
||||
|
||||
;; Dummy identities. Fixed rather than generated: a gate failure should read
|
||||
;; the same on every run.
|
||||
(def ^:private doc-id #uuid "00000000-0000-0000-0000-0000000000d0")
|
||||
(def ^:private page-id #uuid "00000000-0000-0000-0000-0000000000a0")
|
||||
(def ^:private shape-id #uuid "00000000-0000-0000-0000-0000000000b0")
|
||||
(def ^:private frame-id #uuid "00000000-0000-0000-0000-0000000000c0")
|
||||
(def ^:private component-id #uuid "00000000-0000-0000-0000-0000000000e0")
|
||||
|
||||
(def ^:private child-edge
|
||||
{:from-table "Rectangle" :from-id shape-id
|
||||
:to-table "Page" :to-id page-id
|
||||
:position 3})
|
||||
|
||||
(def ^:private ^:dynamic *conn* nil)
|
||||
|
||||
(defn- with-graph-connection
|
||||
"Open a `:memory:` database, create the live schema, run the tests on it.
|
||||
|
||||
Nothing is executed against it — the gate only prepares — but the DDL has to
|
||||
be there for the binder to resolve tables and columns against."
|
||||
[next]
|
||||
(ladybug/with-connection! ":memory:"
|
||||
(fn [conn]
|
||||
(ladybug/exec-on-connection! conn (nodes/ddl-statements))
|
||||
(binding [*conn* conn]
|
||||
(next)))))
|
||||
|
||||
(t/use-fixtures :once with-graph-connection)
|
||||
|
||||
(defn- gate
|
||||
"Assert `statement` binds, and that the engine agrees on read/write."
|
||||
[label statement read-only?]
|
||||
(let [result (ladybug/validate-on-connection! *conn* statement)]
|
||||
(t/is (:ok? result)
|
||||
(str label " does not bind: " (:error result) "\n " statement))
|
||||
(when (:ok? result)
|
||||
(t/is (= read-only? (:read-only? result))
|
||||
(str label " read-only? " (:read-only? result) ", expected " read-only?)))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; the eleven sync templates
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(t/deftest create-node-binds
|
||||
(gate "create-node-statement"
|
||||
(create-node-statement "Rectangle" {:id shape-id
|
||||
:name "a shape"
|
||||
:opacity 1.0
|
||||
:hidden false})
|
||||
false))
|
||||
|
||||
(t/deftest delete-node-binds
|
||||
(gate "delete-node-statement"
|
||||
(delete-node-statement "Rectangle" shape-id)
|
||||
false))
|
||||
|
||||
(t/deftest create-edge-binds
|
||||
(gate "create-edge-statement"
|
||||
(create-edge-statement child-edge)
|
||||
false))
|
||||
|
||||
(t/deftest delete-edge-binds
|
||||
(gate "delete-edge-statement"
|
||||
(delete-edge-statement (dissoc child-edge :position))
|
||||
false))
|
||||
|
||||
(t/deftest set-edge-position-binds
|
||||
(gate "set-edge-position-statement"
|
||||
(set-edge-position-statement child-edge)
|
||||
false))
|
||||
|
||||
(t/deftest create-instance-of-binds
|
||||
(gate "create-instance-of-statement"
|
||||
(create-instance-of-statement frame-id component-id)
|
||||
false))
|
||||
|
||||
(t/deftest delete-instance-of-binds
|
||||
(gate "delete-instance-of-statement"
|
||||
(delete-instance-of-statement frame-id)
|
||||
false))
|
||||
|
||||
(t/deftest set-node-attr-binds
|
||||
(gate "set-node-attr-statement"
|
||||
(set-node-attr-statement "Rectangle" shape-id :name "a shape")
|
||||
false))
|
||||
|
||||
(t/deftest set-page-name-binds
|
||||
(gate "set-page-name-statement"
|
||||
(set-page-name-statement page-id "a page")
|
||||
false))
|
||||
|
||||
(t/deftest remove-node-attr-binds
|
||||
(gate "remove-node-attr-statement"
|
||||
(remove-node-attr-statement "Rectangle" shape-id :name)
|
||||
false))
|
||||
|
||||
(t/deftest set-document-revision-binds
|
||||
(gate "set-document-revision-statement"
|
||||
(set-document-revision-statement doc-id 42)
|
||||
false))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; label quoting across the registry
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(t/deftest every-node-label-binds
|
||||
;; `Group` and `Boolean` are reserved words: unquoted they do not parse.
|
||||
;; One MATCH per registered table is the cheapest way to keep `match-label`
|
||||
;; honest as tables come and go.
|
||||
(doseq [table (map :table nodes/node-types)]
|
||||
(gate (str "delete-node-statement on " table)
|
||||
(delete-node-statement table shape-id)
|
||||
false)))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; the gate itself
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(t/deftest read-only-discriminates
|
||||
;; Without this the `read-only? false` assertions above would hold for a
|
||||
;; `validate-on-connection!` that always answered false.
|
||||
(gate "a read query" "MATCH (n:Rectangle) RETURN count(n);" true))
|
||||
|
||||
(t/deftest bad-statement-is-reported-not-thrown
|
||||
(let [result (ladybug/validate-on-connection!
|
||||
*conn* "MATCH (n:Rectangle) SET n.no_such_column = 1;")]
|
||||
(t/is (false? (:ok? result)))
|
||||
(t/is (string? (:error result)))
|
||||
(t/is (nil? (:read-only? result)))))
|
||||
@@ -0,0 +1,280 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.graph-sync-parity-test
|
||||
"Cold projection and incremental sync are two implementations of one mapping,
|
||||
and this namespace holds them to it.
|
||||
|
||||
`app.graph.projection.document/projection-data` reads a whole file and produces
|
||||
the whole graph. `app.graph.sync/apply-changes!` takes the change vocabulary
|
||||
the editor emits and mutates an already open graph. A graph the second one
|
||||
maintained must equal a graph the first one would build from the same file,
|
||||
or the console shows a graph no rebuild reproduces.
|
||||
|
||||
The round trip: project a file cold into A, apply a change list to A and the
|
||||
same list to the file data, project the resulting data cold into B, and diff
|
||||
A against B. Two `:memory:` databases, no Postgres, no session."
|
||||
(:require
|
||||
[app.common.features :as ffeat]
|
||||
[app.common.files.changes :as cfc]
|
||||
[app.common.time :as ct]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.shape :as cts]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.graph.arrow :as arrow]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.projection.document :as projection.document]
|
||||
[app.graph.projection.transforms :as projection.transforms]
|
||||
[app.graph.schema.nodes :as nodes]
|
||||
[app.graph.sync :as sync]
|
||||
[clojure.test :as t]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; the fixture file
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; Fixed ids: a failure should read the same on every run.
|
||||
(def ^:private file-id #uuid "00000000-0000-0000-0000-00000000f11e")
|
||||
(def ^:private page-id #uuid "00000000-0000-0000-0000-0000000000a1")
|
||||
(def ^:private page2-id #uuid "00000000-0000-0000-0000-0000000000a2")
|
||||
(def ^:private frame-id #uuid "00000000-0000-0000-0000-0000000000f1")
|
||||
(def ^:private rect-id #uuid "00000000-0000-0000-0000-0000000000b1")
|
||||
(def ^:private circ-id #uuid "00000000-0000-0000-0000-0000000000b2")
|
||||
(def ^:private text-id #uuid "00000000-0000-0000-0000-0000000000b3")
|
||||
(def ^:private rect2-id #uuid "00000000-0000-0000-0000-0000000000b4")
|
||||
|
||||
(def ^:private base-revn 1)
|
||||
|
||||
(defn- file-row
|
||||
"The `file` map the projection reads, as `bfc/get-file` returns it minus the
|
||||
data blob."
|
||||
[revn]
|
||||
{:id file-id
|
||||
:name "graph sync parity fixture"
|
||||
:revn revn
|
||||
:version 70
|
||||
:features #{"components/v2"}
|
||||
:created-at (ct/inst "2026-01-01T00:00:00Z")
|
||||
:modified-at (ct/inst "2026-01-02T00:00:00Z")})
|
||||
|
||||
(defn- base-data
|
||||
[]
|
||||
(binding [ffeat/*current* #{"components/v2"}]
|
||||
(ctf/make-file-data file-id page-id)))
|
||||
|
||||
(defn- shape
|
||||
[id type attrs]
|
||||
(cts/setup-shape (merge {:id id
|
||||
:type type
|
||||
:frame-id uuid/zero
|
||||
:parent-id uuid/zero}
|
||||
attrs)))
|
||||
|
||||
(def ^:private changes
|
||||
"One change of every kind the sync path claims to support that this fixture
|
||||
can exercise, in the order an editing session would emit them.
|
||||
|
||||
Four siblings in one container, then a reorder, a reparent, and a delete:
|
||||
sibling order is where the two paths are easiest to get wrong, because the
|
||||
stored `:shapes` list and `IsChildOf.position` run opposite ways."
|
||||
[{:type :add-obj :page-id page-id :id frame-id
|
||||
:parent-id uuid/zero :frame-id uuid/zero
|
||||
:obj (shape frame-id :frame {:name "Board" :width 400 :height 300})}
|
||||
|
||||
{:type :add-obj :page-id page-id :id rect-id
|
||||
:parent-id frame-id :frame-id frame-id
|
||||
:obj (shape rect-id :rect {:name "Rect" :parent-id frame-id :frame-id frame-id
|
||||
:width 100 :height 50})}
|
||||
|
||||
{:type :add-obj :page-id page-id :id circ-id
|
||||
:parent-id frame-id :frame-id frame-id
|
||||
:obj (shape circ-id :circle {:name "Circle" :parent-id frame-id :frame-id frame-id
|
||||
:width 40 :height 40})}
|
||||
|
||||
{:type :add-obj :page-id page-id :id text-id
|
||||
:parent-id frame-id :frame-id frame-id
|
||||
:obj (shape text-id :text {:name "Label" :parent-id frame-id :frame-id frame-id})}
|
||||
|
||||
{:type :add-obj :page-id page-id :id rect2-id
|
||||
:parent-id frame-id :frame-id frame-id
|
||||
:obj (shape rect2-id :rect {:name "Rect two" :parent-id frame-id :frame-id frame-id
|
||||
:width 20 :height 20})}
|
||||
|
||||
;; A rename, and two attributes whose values are falsy: `blocked false` and
|
||||
;; `opacity 0` are values, not absences, on both paths.
|
||||
{:type :mod-obj :page-id page-id :id rect-id
|
||||
:operations [{:type :set :attr :name :val "Renamed rect"}
|
||||
{:type :set :attr :blocked :val false}
|
||||
{:type :set :attr :opacity :val 0}]}
|
||||
|
||||
;; Reorder inside the same container: the edge keeps its endpoints and
|
||||
;; every sibling it passes has to move.
|
||||
{:type :mov-objects :page-id page-id :parent-id frame-id :index 0 :shapes [circ-id]}
|
||||
|
||||
;; Reparent to the page's root frame: the edge moves, and so do the
|
||||
;; shape's own `parent_id` and `frame_id`.
|
||||
{:type :mov-objects :page-id page-id :parent-id uuid/zero :index 0 :shapes [text-id]}
|
||||
|
||||
;; Delete with survivors: the gap in the sibling numbering has to close.
|
||||
{:type :del-obj :page-id page-id :id rect-id}
|
||||
|
||||
{:type :add-page :id page2-id :name "Page two"}
|
||||
{:type :mod-page :id page-id :name "Page one, renamed"}])
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; projecting and reading back
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- load-graph!
|
||||
"Create the schema on `conn`, project `data` into it, run the transforms.
|
||||
|
||||
Returns the projection, which is also what the sync index is built from."
|
||||
[conn data file]
|
||||
(let [projection (projection.document/projection-data data file)]
|
||||
(ladybug/exec-on-connection! conn (nodes/ddl-statements))
|
||||
(arrow/with-allocator!
|
||||
(fn [allocator] (arrow/load-projection! conn projection allocator)))
|
||||
(projection.transforms/apply-transforms! nil conn data file)
|
||||
projection))
|
||||
|
||||
(defn- rel-tables
|
||||
[conn]
|
||||
(mapv first (:rows (ladybug/query-on-connection!
|
||||
conn "CALL show_tables() WHERE type = 'REL' RETURN name;"
|
||||
:max-rows 1000))))
|
||||
|
||||
(defn- rel-properties
|
||||
"Property names on rel table `rel`, in catalog order."
|
||||
[conn rel]
|
||||
(mapv (comp str second)
|
||||
(:rows (ladybug/query-on-connection!
|
||||
conn (str "CALL table_info('" rel "') RETURN *;")
|
||||
:max-rows 1000))))
|
||||
|
||||
(defn- node-rows
|
||||
[conn table]
|
||||
(:rows (ladybug/query-on-connection!
|
||||
conn (str "MATCH (n:" (nodes/match-label table) ") RETURN n.* ORDER BY n.id;")
|
||||
:max-rows 100000)))
|
||||
|
||||
(defn- edge-rows
|
||||
[conn rel props]
|
||||
(let [returns (into ["a.id" "b.id"] (map #(str "r.`" % "`")) props)]
|
||||
(:rows (ladybug/query-on-connection!
|
||||
conn (str "MATCH (a)-[r:`" rel "`]->(b) "
|
||||
"RETURN " (clojure.string/join ", " returns) " "
|
||||
"ORDER BY a.id, b.id;")
|
||||
:max-rows 100000))))
|
||||
|
||||
(defn- keyed-rows
|
||||
"Rows as `{key {column value}}`, so a difference names a row and a column.
|
||||
|
||||
Values are stringified: both connections hand a value back through the same
|
||||
reader, so any difference in the strings is a difference in the graph."
|
||||
[columns key-columns rows]
|
||||
(into {}
|
||||
(map (fn [row]
|
||||
(let [cells (zipmap columns (map str row))]
|
||||
[(mapv cells key-columns) cells])))
|
||||
rows))
|
||||
|
||||
(defn- snapshot
|
||||
"Every node row and every edge row in the database, keyed by table."
|
||||
[conn]
|
||||
{:nodes (into {}
|
||||
(map (fn [{:keys [table]}]
|
||||
(let [columns (nodes/columns table)]
|
||||
[table (keyed-rows columns ["id"] (node-rows conn table))])))
|
||||
nodes/node-types)
|
||||
:edges (into {}
|
||||
(map (fn [rel]
|
||||
(let [columns (into ["from" "to"] (rel-properties conn rel))]
|
||||
[rel (keyed-rows columns ["from" "to"]
|
||||
(edge-rows conn rel (rel-properties conn rel)))])))
|
||||
(rel-tables conn))})
|
||||
|
||||
(defn- row-diff
|
||||
[rows-a rows-b]
|
||||
(into {}
|
||||
(for [k (sort (into #{} (concat (keys rows-a) (keys rows-b))))
|
||||
:let [a (get rows-a k)
|
||||
b (get rows-b k)]
|
||||
:when (not= a b)]
|
||||
[k (cond
|
||||
(nil? a) {:only-in :rebuilt}
|
||||
(nil? b) {:only-in :synced}
|
||||
:else (into {}
|
||||
(for [c (sort (into #{} (concat (keys a) (keys b))))
|
||||
:when (not= (get a c) (get b c))]
|
||||
[c {:synced (get a c) :rebuilt (get b c)}])))])))
|
||||
|
||||
(defn- diff
|
||||
"Where the two snapshots disagree, down to the row and the column."
|
||||
[a b]
|
||||
(into {}
|
||||
(for [kind [:nodes :edges]
|
||||
table (sort (into #{} (concat (keys (get a kind)) (keys (get b kind)))))
|
||||
:let [d (row-diff (get-in a [kind table]) (get-in b [kind table]))]
|
||||
:when (seq d)]
|
||||
[[kind table] d])))
|
||||
|
||||
(defn- with-two-connections
|
||||
[f]
|
||||
(ladybug/with-connection! ":memory:"
|
||||
(fn [conn-a]
|
||||
(ladybug/with-connection! ":memory:"
|
||||
(fn [conn-b]
|
||||
(f conn-a conn-b))))))
|
||||
|
||||
(defn- round-trip
|
||||
"Sync `change-list` into A, rebuild the same file into B, return the diff."
|
||||
[change-list]
|
||||
(let [data0 (base-data)
|
||||
data1 (cfc/process-changes data0 change-list)
|
||||
revn1 (inc base-revn)]
|
||||
(with-two-connections
|
||||
(fn [conn-a conn-b]
|
||||
(let [projection (load-graph! conn-a data0 (file-row base-revn))
|
||||
index (sync/build-index file-id base-revn projection)
|
||||
result (sync/apply-changes! conn-a index change-list revn1)]
|
||||
(load-graph! conn-b data1 (file-row revn1))
|
||||
{:diff (diff (snapshot conn-a) (snapshot conn-b))
|
||||
:applied (:applied result)
|
||||
:skipped (:skipped result)})))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; the tests
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(t/deftest every-change-in-the-list-is-supported
|
||||
(let [{:keys [applied skipped]} (round-trip changes)]
|
||||
(t/is (empty? skipped)
|
||||
(str "the fixture must exercise the sync path, not the skip path: " (pr-str skipped)))
|
||||
(t/is (= (count changes) (count applied)))))
|
||||
|
||||
(t/deftest synced-graph-equals-rebuilt-graph
|
||||
(let [{:keys [diff]} (round-trip changes)]
|
||||
(t/is (empty? diff)
|
||||
(str "cold projection and sync replay disagree on "
|
||||
(pr-str (keys diff)) "\n" (pr-str diff)))))
|
||||
|
||||
(t/deftest the-diff-catches-an-injected-sync-bug
|
||||
;; The round trip is only worth running if it fails when sync is wrong.
|
||||
;; `apply-mov-objects` maintains `IsChildOf`; drop the change from the list
|
||||
;; sync sees, keep it in the list the file sees, and the edge must differ.
|
||||
(let [data0 (base-data)
|
||||
data1 (cfc/process-changes data0 changes)
|
||||
crippled (remove #(= :mov-objects (:type %)) changes)
|
||||
revn1 (inc base-revn)
|
||||
result (with-two-connections
|
||||
(fn [conn-a conn-b]
|
||||
(let [projection (load-graph! conn-a data0 (file-row base-revn))
|
||||
index (sync/build-index file-id base-revn projection)]
|
||||
(sync/apply-changes! conn-a index crippled revn1)
|
||||
(load-graph! conn-b data1 (file-row revn1))
|
||||
(diff (snapshot conn-a) (snapshot conn-b)))))]
|
||||
(t/is (contains? result [:edges "IsChildOf"])
|
||||
"a sync that skips a reparent must show up as an IsChildOf difference")))
|
||||
@@ -0,0 +1,51 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.passwords-test
|
||||
(:require
|
||||
[app.auth.passwords :as passwords]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(defn- run-validation
|
||||
[password]
|
||||
(try
|
||||
(passwords/validate-password password)
|
||||
nil
|
||||
(catch Throwable e
|
||||
e)))
|
||||
|
||||
(t/deftest validate-password-accepts-strong-password
|
||||
(t/is (nil? (run-validation "Str0ng!Pass"))))
|
||||
|
||||
(t/deftest validate-password-rejects-too-short-password
|
||||
(let [error (run-validation "Ab1!x")]
|
||||
(t/is (th/ex-of-code? error :weak-password))
|
||||
(t/is (= ["errors.weak-password.too-short"] (:details (ex-data error))))))
|
||||
|
||||
(t/deftest validate-password-rejects-missing-lowercase
|
||||
(let [error (run-validation "ABCDEFG1!")]
|
||||
(t/is (th/ex-of-code? error :weak-password))
|
||||
(t/is (= ["errors.weak-password.insufficient-lowercase"]
|
||||
(:details (ex-data error))))))
|
||||
|
||||
(t/deftest validate-password-rejects-missing-uppercase
|
||||
(let [error (run-validation "abcdefg1!")]
|
||||
(t/is (th/ex-of-code? error :weak-password))
|
||||
(t/is (= ["errors.weak-password.insufficient-uppercase"]
|
||||
(:details (ex-data error))))))
|
||||
|
||||
(t/deftest validate-password-rejects-missing-digit
|
||||
(let [error (run-validation "Abcdefgh!")]
|
||||
(t/is (th/ex-of-code? error :weak-password))
|
||||
(t/is (= ["errors.weak-password.insufficient-digits"]
|
||||
(:details (ex-data error))))))
|
||||
|
||||
(t/deftest validate-password-rejects-missing-special
|
||||
(let [error (run-validation "Abcdefgh1")]
|
||||
(t/is (th/ex-of-code? error :weak-password))
|
||||
(t/is (= ["errors.weak-password.insufficient-special"]
|
||||
(:details (ex-data error))))))
|
||||
@@ -6,19 +6,11 @@
|
||||
|
||||
(ns backend-tests.rpc-auth-test
|
||||
(:require
|
||||
[app.auth.ldap :as ldap]
|
||||
[app.auth.login-lockout :as lol]
|
||||
[app.common.flags :as flags]
|
||||
[app.common.generic-pool :as gpool]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.http.session :as session]
|
||||
[app.redis :as rds]
|
||||
[app.rpc.commands.ldap :as ldap-cmd]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
[yetti.response :as yres])
|
||||
(:import
|
||||
java.lang.AutoCloseable))
|
||||
[yetti.response :as yres]))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
@@ -111,122 +103,4 @@
|
||||
(t/is (nil? (session/read-session manager sid))
|
||||
"replayed token must not resolve to a valid session after logout")))
|
||||
|
||||
(defn- cleanup-redis
|
||||
[profile-id]
|
||||
(let [pool (get-in th/*system* [:app.redis/pool])
|
||||
conn (gpool/get pool)]
|
||||
(try
|
||||
(rds/del @conn (str "penpot.login-lockout." profile-id))
|
||||
(finally
|
||||
(.close ^AutoCloseable conn)))))
|
||||
|
||||
(t/deftest login-with-password-locks-after-max-attempts
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [:enable-account-lockout])]
|
||||
(let [profile (th/create-profile* 200 {:is-active true})]
|
||||
(cleanup-redis (:id profile))
|
||||
;; 4 failed attempts — not yet locked
|
||||
(dotimes [i 4]
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}
|
||||
out (th/command! data)]
|
||||
(t/is (th/ex-of-code? (:error out) :wrong-credentials))))
|
||||
;; 5th attempt — should trigger lockout
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}
|
||||
out (th/command! data)]
|
||||
(t/is (th/ex-of-type? (:error out) :rate-limit))
|
||||
(t/is (th/ex-of-code? (:error out) :account-locked))))))
|
||||
|
||||
(t/deftest login-with-password-returns-rate-limit-when-locked
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [:enable-account-lockout])]
|
||||
(let [profile (th/create-profile* 201 {:is-active true})]
|
||||
(cleanup-redis (:id profile))
|
||||
;; Trigger lockout
|
||||
(dotimes [i 5]
|
||||
(th/command! {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}))
|
||||
;; Next attempt with correct password — still blocked
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "Test123!"}
|
||||
out (th/command! data)]
|
||||
(t/is (th/ex-of-type? (:error out) :rate-limit))
|
||||
(t/is (th/ex-of-code? (:error out) :account-locked))
|
||||
(let [ttl (:ttl (ex-data (:error out)))]
|
||||
(t/is (some? ttl) "ttl should be present in ex-data")
|
||||
(t/is (pos? ttl) "ttl should be a positive number"))))))
|
||||
|
||||
(t/deftest successful-login-clears-failed-attempts
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [:enable-account-lockout])]
|
||||
(let [profile (th/create-profile* 202 {:is-active true})]
|
||||
(cleanup-redis (:id profile))
|
||||
;; 3 failed attempts
|
||||
(dotimes [i 3]
|
||||
(th/command! {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}))
|
||||
;; Successful login
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "Test123!"}
|
||||
out (th/command! data)]
|
||||
(t/is (nil? (:error out))))
|
||||
;; 4 more failed attempts should NOT trigger lockout (counter was reset)
|
||||
(dotimes [i 4]
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}
|
||||
out (th/command! data)]
|
||||
(t/is (th/ex-of-code? (:error out) :wrong-credentials)))))))
|
||||
|
||||
(t/deftest lockout-does-not-apply-when-flag-disabled
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [])]
|
||||
(let [profile (th/create-profile* 203 {:is-active true})]
|
||||
(cleanup-redis (:id profile))
|
||||
;; 10 failed attempts — no lockout without the flag
|
||||
(dotimes [i 10]
|
||||
(let [data {::th/type :login-with-password
|
||||
:email (:email profile)
|
||||
:password "wrongpassword"}
|
||||
out (th/command! data)]
|
||||
(t/is (th/ex-of-code? (:error out) :wrong-credentials)))))))
|
||||
|
||||
(t/deftest login-with-ldap-returns-rate-limit-when-locked
|
||||
"Verify that login-with-ldap short-circuits when account is locked,
|
||||
without attempting LDAP authentication. Calls the actual command
|
||||
function with a mock provider to test the full flow."
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [:enable-account-lockout :login-with-ldap])
|
||||
ldap/authenticate (constantly nil)]
|
||||
(let [profile (th/create-profile* 204 {:is-active true})
|
||||
cfg (assoc th/*system* ::ldap/provider {})]
|
||||
(cleanup-redis (:id profile))
|
||||
;; Lock account with 5 failed attempts
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! cfg (:id profile)))
|
||||
;; Call actual command function — should be blocked without calling authenticate
|
||||
(let [out (th/try-on! (#'ldap-cmd/sm$login-with-ldap
|
||||
cfg
|
||||
{:email (:email profile)
|
||||
:password "wrongpassword"}))]
|
||||
(t/is (th/ex-of-type? (:error out) :rate-limit))
|
||||
(t/is (th/ex-of-code? (:error out) :account-locked))))))
|
||||
|
||||
(t/deftest recover-profile-clears-lockout
|
||||
"Verify that clearing attempts (e.g. after password reset) unlocks the account."
|
||||
(with-redefs [app.config/flags (flags/parse flags/default [:enable-account-lockout])]
|
||||
(let [profile (th/create-profile* 205 {:is-active true})
|
||||
cfg th/*system*]
|
||||
(cleanup-redis (:id profile))
|
||||
;; Lock the account
|
||||
(dotimes [i 5]
|
||||
(lol/record-failed-attempt! cfg (:id profile)))
|
||||
(t/is (:locked? (lol/locked? cfg (:id profile))))
|
||||
;; Simulate password reset clearing
|
||||
(lol/clear-attempts! cfg (:id profile))
|
||||
;; Verify unlocked
|
||||
(t/is (false? (:locked? (lol/locked? cfg (:id profile))))))))
|
||||
|
||||
|
||||
@@ -285,3 +285,196 @@
|
||||
|
||||
(let [threads (th/db-query :comment-thread {:file-id (:id file-1)})]
|
||||
(t/is (= 0 (count threads)))))))))
|
||||
|
||||
(t/deftest share-link-who-comment-team-cannot-comment
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
outsider (th/create-profile* 2 {:is-active true})
|
||||
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})
|
||||
project (th/create-project* 1 {:team-id (:id team)
|
||||
:profile-id (:id owner)})
|
||||
file (th/create-file* 1 {:profile-id (:id owner)
|
||||
:project-id (:id project)})
|
||||
page-id (get-in file [:data :pages 0])
|
||||
|
||||
share (th/command! {::th/type :create-share-link
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:pages #{page-id}
|
||||
:who-comment "team"
|
||||
:who-inspect "all"})
|
||||
share-id (get-in share [:result :id])]
|
||||
|
||||
(t/testing "outsider with who-comment=team share-link cannot get-comment-threads"
|
||||
(let [out (th/command! {::th/type :get-comment-threads
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:share-id share-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :not-found (th/ex-type (:error out))))))
|
||||
|
||||
(t/testing "outsider with who-comment=team share-link cannot create-comment-thread"
|
||||
(let [out (th/command! {::th/type :create-comment-thread
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:page-id page-id
|
||||
:position (gpt/point 0)
|
||||
:content "outsider comment"
|
||||
:frame-id uuid/zero
|
||||
:share-id share-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :not-found (th/ex-type (:error out))))))))
|
||||
|
||||
(t/deftest share-link-who-comment-all-can-comment
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
outsider (th/create-profile* 2 {:is-active true})
|
||||
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})
|
||||
project (th/create-project* 1 {:team-id (:id team)
|
||||
:profile-id (:id owner)})
|
||||
file (th/create-file* 1 {:profile-id (:id owner)
|
||||
:project-id (:id project)})
|
||||
page-id (get-in file [:data :pages 0])
|
||||
|
||||
share (th/command! {::th/type :create-share-link
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:pages #{page-id}
|
||||
:who-comment "all"
|
||||
:who-inspect "all"})
|
||||
share-id (get-in share [:result :id])]
|
||||
|
||||
(t/testing "outsider with who-comment=all share-link can get-comment-threads"
|
||||
(let [out (th/command! {::th/type :get-comment-threads
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:share-id share-id})]
|
||||
(t/is (th/success? out))))
|
||||
|
||||
(t/testing "outsider with who-comment=all share-link can create-comment-thread"
|
||||
(let [out (th/command! {::th/type :create-comment-thread
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:page-id page-id
|
||||
:position (gpt/point 0)
|
||||
:content "outsider comment"
|
||||
:frame-id uuid/zero
|
||||
:share-id share-id})]
|
||||
(t/is (th/success? out))))))
|
||||
|
||||
(t/deftest share-link-page-scope-enforced
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
outsider (th/create-profile* 2 {:is-active true})
|
||||
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})
|
||||
project (th/create-project* 1 {:team-id (:id team)
|
||||
:profile-id (:id owner)})
|
||||
file (th/create-file* 1 {:profile-id (:id owner)
|
||||
:project-id (:id project)})
|
||||
|
||||
page-a (get-in file [:data :pages 0])
|
||||
page-b (uuid/random)
|
||||
|
||||
_ (th/command! {::th/type :update-file
|
||||
::rpc/profile-id (:id owner)
|
||||
:id (:id file)
|
||||
:session-id (uuid/random)
|
||||
:revn 0
|
||||
:vern 0
|
||||
:changes [{:type :add-page
|
||||
:id page-b
|
||||
:page {:id page-b
|
||||
:name "Page B"
|
||||
:options {}
|
||||
:objects {}}}]})
|
||||
|
||||
thread-a (th/command! {::th/type :create-comment-thread
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:page-id page-a
|
||||
:position (gpt/point 0)
|
||||
:content "comment on page A"
|
||||
:frame-id uuid/zero})
|
||||
thread-b (th/command! {::th/type :create-comment-thread
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:page-id page-b
|
||||
:position (gpt/point 0)
|
||||
:content "comment on page B"
|
||||
:frame-id uuid/zero})
|
||||
|
||||
thread-a-id (get-in thread-a [:result :id])
|
||||
thread-b-id (get-in thread-b [:result :id])
|
||||
|
||||
share (th/command! {::th/type :create-share-link
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:pages #{page-a}
|
||||
:who-comment "all"
|
||||
:who-inspect "all"})
|
||||
share-id (get-in share [:result :id])]
|
||||
|
||||
(t/testing "share-link holder can get-comment-threads for shared page only"
|
||||
(let [out (th/command! {::th/type :get-comment-threads
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:share-id share-id})
|
||||
result (:result out)]
|
||||
(t/is (th/success? out))
|
||||
(t/is (= 1 (count result)))
|
||||
(t/is (= page-a (:page-id (first result))))))
|
||||
|
||||
(t/testing "share-link holder cannot get-comment-thread for unshared page"
|
||||
(let [out (th/command! {::th/type :get-comment-thread
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:id thread-b-id
|
||||
:share-id share-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :not-found (th/ex-type (:error out))))))
|
||||
|
||||
(t/testing "share-link holder can get-comment-thread for shared page"
|
||||
(let [out (th/command! {::th/type :get-comment-thread
|
||||
::rpc/profile-id (:id outsider)
|
||||
:file-id (:id file)
|
||||
:id thread-a-id
|
||||
:share-id share-id})]
|
||||
(t/is (th/success? out))))
|
||||
|
||||
(t/testing "share-link holder cannot get-comments for thread on unshared page"
|
||||
(let [out (th/command! {::th/type :get-comments
|
||||
::rpc/profile-id (:id outsider)
|
||||
:thread-id thread-b-id
|
||||
:share-id share-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (= :not-found (th/ex-type (:error out))))))))
|
||||
|
||||
(t/deftest membership-can-still-comment
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
member (th/create-profile* 2 {:is-active true})
|
||||
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})
|
||||
_ (th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id member)
|
||||
:role :editor})
|
||||
project (th/create-project* 1 {:team-id (:id team)
|
||||
:profile-id (:id owner)})
|
||||
file (th/create-file* 1 {:profile-id (:id owner)
|
||||
:project-id (:id project)})
|
||||
page-id (get-in file [:data :pages 0])]
|
||||
|
||||
(t/testing "team member can get-comment-threads without share-id"
|
||||
(let [out (th/command! {::th/type :get-comment-threads
|
||||
::rpc/profile-id (:id member)
|
||||
:file-id (:id file)})]
|
||||
(t/is (th/success? out))))
|
||||
|
||||
(t/testing "team member can create-comment-thread without share-id"
|
||||
(let [out (th/command! {::th/type :create-comment-thread
|
||||
::rpc/profile-id (:id member)
|
||||
:file-id (:id file)
|
||||
:page-id page-id
|
||||
:position (gpt/point 0)
|
||||
:content "member comment"
|
||||
:frame-id uuid/zero})]
|
||||
(t/is (th/success? out))))))
|
||||
@@ -0,0 +1,76 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.rpc-demo-test
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.config :as cf]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
|
||||
;; Capture the real verifier before the shared test fixture replaces it.
|
||||
(def verify-password* auth/verify-password)
|
||||
|
||||
(t/deftest weak-password-hash-verifies
|
||||
(let [password "DemoPassword123!"
|
||||
hashed (auth/derive-password-weak password)]
|
||||
(t/is (:valid (verify-password* password hashed)))))
|
||||
|
||||
(t/deftest create-demo-profile-uses-unique-uuid-email
|
||||
(with-redefs [cf/flags (conj cf/flags :demo-users)]
|
||||
(let [first-result (th/command! {::th/type :create-demo-profile})
|
||||
second-result (th/command! {::th/type :create-demo-profile})
|
||||
first-profile (:result first-result)
|
||||
second-profile (:result second-result)]
|
||||
(t/is (nil? (:error first-result)))
|
||||
(t/is (nil? (:error second-result)))
|
||||
(t/is (re-matches #"demo-[0-9a-fA-F-]+@demo\.example\.com"
|
||||
(:email first-profile)))
|
||||
(t/is (not= (:email first-profile) (:email second-profile))))))
|
||||
|
||||
(t/deftest create-demo-profile-requires-feature-flag
|
||||
(with-redefs [cf/flags (disj cf/flags :demo-users)]
|
||||
(let [{:keys [error]} (th/command! {::th/type :create-demo-profile})]
|
||||
(t/is (th/ex-of-code? error :demo-users-not-allowed)))))
|
||||
|
||||
(t/deftest create-demo-profile-keeps-onboarding-by-default
|
||||
(with-redefs [cf/flags (conj cf/flags :demo-users)]
|
||||
(let [{:keys [error result]} (th/command! {::th/type :create-demo-profile})]
|
||||
(t/is (nil? error))
|
||||
(let [saved (th/db-get :profile {:email (:email result)})
|
||||
decoded (profile/decode-row saved)]
|
||||
(t/is (nil? (get-in decoded [:props :onboarding-viewed])))))))
|
||||
|
||||
(t/deftest create-demo-profile-skips-onboarding-when-requested
|
||||
(with-redefs [cf/flags (conj cf/flags :demo-users)]
|
||||
(let [{:keys [error result]} (th/command! {::th/type :create-demo-profile
|
||||
:skip-onboarding true})]
|
||||
(t/is (nil? error))
|
||||
(let [saved (th/db-get :profile {:email (:email result)})
|
||||
decoded (profile/decode-row saved)]
|
||||
(t/is (true? (get-in decoded [:props :onboarding-viewed])))
|
||||
(t/is (= (:main cf/version)
|
||||
(get-in decoded [:props :release-notes-viewed])))))))
|
||||
|
||||
(t/deftest create-demo-profile-explicit-false-keeps-onboarding
|
||||
(with-redefs [cf/flags (conj cf/flags :demo-users)]
|
||||
(let [{:keys [error result]} (th/command! {::th/type :create-demo-profile
|
||||
:skip-onboarding false})]
|
||||
(t/is (nil? error))
|
||||
(let [saved (th/db-get :profile {:email (:email result)})
|
||||
decoded (profile/decode-row saved)]
|
||||
(t/is (nil? (get-in decoded [:props :onboarding-viewed])))))))
|
||||
|
||||
(t/deftest create-demo-profile-rejects-non-boolean-skip-onboarding
|
||||
(with-redefs [cf/flags (conj cf/flags :demo-users)]
|
||||
(let [{:keys [error]} (th/command! {::th/type :create-demo-profile
|
||||
:skip-onboarding "yes"})]
|
||||
(t/is (th/ex-of-type? error :validation))
|
||||
(t/is (th/ex-of-code? error :params-validation)))))
|
||||
@@ -59,8 +59,13 @@
|
||||
(let [out (th/command! {::th/type :create-upload-session
|
||||
::rpc/profile-id (:id prof)
|
||||
:total-chunks total-chunks})]
|
||||
(t/is (nil? (:error out)))
|
||||
(:session-id (:result out))))
|
||||
(let [session-id (:session-id (:result out))]
|
||||
(t/is (nil? (:error out))
|
||||
(str "create-upload-session failed: "
|
||||
(some-> (:error out) ex-data)))
|
||||
(t/is (uuid? session-id)
|
||||
(str "create-upload-session returned an invalid session-id: " session-id))
|
||||
session-id)))
|
||||
|
||||
(defn- upload-font-chunked!
|
||||
"Splits `font-bytes` into chunks of `chunk-size` bytes, creates an upload
|
||||
@@ -68,14 +73,15 @@
|
||||
[prof ^bytes font-bytes mtype chunk-size]
|
||||
(let [chunks (split-bytes-into-chunks font-bytes chunk-size)
|
||||
session-id (create-upload-session! prof (count chunks))]
|
||||
(doseq [[idx chunk-data] (map-indexed vector chunks)]
|
||||
(let [mfile (make-chunk-mfile chunk-data mtype)
|
||||
out (th/command! {::th/type :upload-chunk
|
||||
::rpc/profile-id (:id prof)
|
||||
:session-id session-id
|
||||
:index idx
|
||||
:content mfile})]
|
||||
(t/is (nil? (:error out)))))
|
||||
(when (uuid? session-id)
|
||||
(doseq [[idx chunk-data] (map-indexed vector chunks)]
|
||||
(let [mfile (make-chunk-mfile chunk-data mtype)
|
||||
out (th/command! {::th/type :upload-chunk
|
||||
::rpc/profile-id (:id prof)
|
||||
:session-id session-id
|
||||
:index idx
|
||||
:content mfile})]
|
||||
(t/is (nil? (:error out))))))
|
||||
session-id))
|
||||
|
||||
(defn- assert-font-variant-result
|
||||
@@ -613,43 +619,42 @@
|
||||
;; N2-07: A user with edit permissions on their own team must not be
|
||||
;; able to create a font variant using a font-id that already belongs
|
||||
;; to another team (BOLA / CWE-639).
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof1 (th/create-profile* 1 {:is-active true})
|
||||
prof2 (th/create-profile* 2 {:is-active true})
|
||||
team1 (:default-team-id prof1)
|
||||
team2 (:default-team-id prof2)
|
||||
font-id (uuid/custom 10 999)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.ttf")
|
||||
(io/read*))]
|
||||
(let [prof1 (th/create-profile* 1 {:is-active true})
|
||||
prof2 (th/create-profile* 2 {:is-active true})
|
||||
team1 (:default-team-id prof1)
|
||||
team2 (:default-team-id prof2)
|
||||
font-id (uuid/custom 10 999)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.ttf")
|
||||
(io/read*))]
|
||||
|
||||
;; prof1 creates a font variant in team1 with font-id
|
||||
(let [session-id (upload-font-chunked! prof1 data "font/ttf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof1)
|
||||
:team-id team1
|
||||
:font-id font-id
|
||||
:font-family "SharedFont"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (nil? (:error out))))
|
||||
;; prof1 creates a font variant in team1 with font-id
|
||||
(let [session-id (upload-font-chunked! prof1 data "font/ttf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof1)
|
||||
:team-id team1
|
||||
:font-id font-id
|
||||
:font-family "SharedFont"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (nil? (:error out))))
|
||||
|
||||
;; prof2 tries to create a variant using the same font-id but
|
||||
;; in team2, which must be rejected because font-id belongs to team1
|
||||
(let [session-id (upload-font-chunked! prof2 data "font/ttf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof2)
|
||||
:team-id team2
|
||||
:font-id font-id
|
||||
:font-family "SharedFont"
|
||||
:font-weight 700
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :not-found (-> out :error ex-data :type)))
|
||||
(t/is (= :object-not-found (-> out :error ex-data :code)))))))
|
||||
;; prof2 tries to create a variant using the same font-id but
|
||||
;; in team2, which must be rejected because font-id belongs to team1
|
||||
(let [session-id (upload-font-chunked! prof2 data "font/ttf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof2)
|
||||
:team-id team2
|
||||
:font-id font-id
|
||||
:font-family "SharedFont"
|
||||
:font-weight 700
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :not-found (-> out :error ex-data :type)))
|
||||
(t/is (= :object-not-found (-> out :error ex-data :code))))))
|
||||
|
||||
(t/deftest get-font-variants-nonexistent-file
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
|
||||
@@ -194,6 +194,15 @@
|
||||
(string? (get version k)))))
|
||||
(t/is (= cf/version version))))
|
||||
|
||||
(t/deftest get-air-gapped
|
||||
(let [out (th/management-command! {::th/type :get-air-gapped})]
|
||||
(t/is (th/success? out))
|
||||
(t/is (false? (-> out :result :air-gapped))))
|
||||
(binding [cf/flags (conj cf/flags :air-gapped-conf)]
|
||||
(let [out (th/management-command! {::th/type :get-air-gapped})]
|
||||
(t/is (th/success? out))
|
||||
(t/is (true? (-> out :result :air-gapped))))))
|
||||
|
||||
(t/deftest get-teams-returns-only-owned-non-default-non-deleted
|
||||
(with-mocks [nitrate-mock {:target 'app.nitrate/call :return nil}]
|
||||
(let [profile (th/create-profile* 1 {:is-active true})
|
||||
@@ -268,7 +277,7 @@
|
||||
new-team (th/db-get :team {:id new-team-id})]
|
||||
(t/is (th/success? out))
|
||||
(t/is (= 1 (count (set/difference after-teams before-teams))))
|
||||
(t/is (= "Your Penpot" (:name new-team)))
|
||||
(t/is (= "Personal Projects" (:name new-team)))
|
||||
(t/is (true? (:is-default new-team))))))
|
||||
|
||||
(t/deftest get-managed-profiles-returns-unique-members-for-owned-teams
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns backend-tests.rpc-plugins-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
|
||||
(def ^:private plugin-id-1 (str (uuid/next)))
|
||||
(def ^:private plugin-id-2 (str (uuid/next)))
|
||||
|
||||
(def ^:private valid-plugin
|
||||
{:plugin-id plugin-id-1
|
||||
:name "Test Plugin"
|
||||
:description "A test plugin"
|
||||
:host "https://example.com"
|
||||
:code "(function() { console.log('hello'); })()"
|
||||
:icon "icon.svg"
|
||||
:permissions #{"content:read" "content:write"}})
|
||||
|
||||
(t/deftest add-profile-plugin-accepts-valid-permissions
|
||||
(let [profile (th/create-profile* 1)
|
||||
data {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin valid-plugin}
|
||||
out (th/command! data)]
|
||||
|
||||
(t/is (nil? (:error out)))
|
||||
(t/is (some? (:result out)))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (= [plugin-id-1] (:ids plugins)))
|
||||
(t/is (= valid-plugin (get-in plugins [:data plugin-id-1]))))))
|
||||
|
||||
(t/deftest add-profile-plugin-rejects-invalid-permissions
|
||||
(let [profile (th/create-profile* 1)
|
||||
plugin (assoc valid-plugin :permissions #{"content:read" "admin:delete"})
|
||||
data {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin plugin}
|
||||
out (th/command! data)]
|
||||
|
||||
;; Schema validation catches invalid permissions before custom validation
|
||||
(t/is (th/ex-info? (:error out)))
|
||||
(t/is (th/ex-of-type? (:error out) :validation))
|
||||
(t/is (th/ex-of-code? (:error out) :params-validation))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (nil? plugins) "No plugins should be persisted when validation fails"))))
|
||||
|
||||
(t/deftest add-profile-plugin-updates-existing-plugin
|
||||
(let [profile (th/create-profile* 1)
|
||||
data1 {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin valid-plugin}
|
||||
_ (th/command! data1)
|
||||
|
||||
updated-plugin (assoc valid-plugin :name "Updated Plugin")
|
||||
data2 {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin updated-plugin}
|
||||
out (th/command! data2)]
|
||||
|
||||
(t/is (nil? (:error out)))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (= 1 (count (:ids plugins))) "Should still have only one plugin")
|
||||
(t/is (= "Updated Plugin" (get-in plugins [:data plugin-id-1 :name]))))))
|
||||
|
||||
(t/deftest remove-profile-plugin-removes-plugin
|
||||
(let [profile (th/create-profile* 1)
|
||||
data1 {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin valid-plugin}
|
||||
_ (th/command! data1)
|
||||
|
||||
data2 {::th/type :remove-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin-id (uuid/uuid plugin-id-1)}
|
||||
out (th/command! data2)]
|
||||
|
||||
(t/is (nil? (:error out)))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (= [] (:ids plugins)))
|
||||
(t/is (empty? (:data plugins))))))
|
||||
|
||||
(t/deftest remove-profile-plugin-handles-nonexistent-plugin
|
||||
(let [profile (th/create-profile* 1)
|
||||
data {::th/type :remove-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin-id (uuid/next)}
|
||||
out (th/command! data)]
|
||||
|
||||
(t/is (nil? (:error out)))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (or (nil? plugins)
|
||||
(and (empty? (:ids plugins))
|
||||
(empty? (:data plugins))))
|
||||
"Plugins should be nil or empty when no plugins exist"))))
|
||||
|
||||
(t/deftest add-profile-plugin-multiple-plugins
|
||||
(let [profile (th/create-profile* 1)
|
||||
plugin1 valid-plugin
|
||||
plugin2 (assoc valid-plugin
|
||||
:plugin-id plugin-id-2
|
||||
:name "Second Plugin")
|
||||
|
||||
data1 {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin plugin1}
|
||||
_ (th/command! data1)
|
||||
|
||||
data2 {::th/type :add-profile-plugin
|
||||
::rpc/profile-id (:id profile)
|
||||
:plugin plugin2}
|
||||
_ (th/command! data2)]
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)
|
||||
plugins (get-in props [:props :plugins])]
|
||||
(t/is (= 2 (count (:ids plugins))))
|
||||
(t/is (contains? (set (:ids plugins)) plugin-id-1))
|
||||
(t/is (contains? (set (:ids plugins)) plugin-id-2))
|
||||
(t/is (= "Test Plugin" (get-in plugins [:data plugin-id-1 :name])))
|
||||
(t/is (= "Second Plugin" (get-in plugins [:data plugin-id-2 :name]))))))
|
||||
|
||||
(t/deftest update-profile-props-rejects-plugins
|
||||
(let [profile (th/create-profile* 1)
|
||||
data {::th/type :update-profile-props
|
||||
::rpc/profile-id (:id profile)
|
||||
:props {:plugins {:ids ["test"] :data {"test" valid-plugin}}}}
|
||||
out (th/command! data)]
|
||||
|
||||
(t/is (th/ex-info? (:error out)))
|
||||
(t/is (th/ex-of-type? (:error out) :validation))
|
||||
(t/is (th/ex-of-code? (:error out) :params-validation))
|
||||
|
||||
(let [saved (th/db-get :profile {:id (:id profile)})
|
||||
props (profile/decode-row saved)]
|
||||
(t/is (nil? (get-in props [:props :plugins]))
|
||||
":plugins must not be writable via update-profile-props"))))
|
||||
@@ -424,6 +424,38 @@
|
||||
(let [count-after (:count (th/db-exec-one! ["SELECT count(*) FROM http_session_v2 WHERE profile_id = ?" (:id prof)]))]
|
||||
(t/is (= 0 count-after)))))
|
||||
|
||||
(t/deftest profile-deletion-via-gc-cascades
|
||||
(let [prof (th/create-profile* 1)
|
||||
file (th/create-file* 1 {:profile-id (:id prof)
|
||||
:project-id (:default-project-id prof)
|
||||
:is-shared false})
|
||||
team-id (:default-team-id prof)
|
||||
project-id (:default-project-id prof)
|
||||
file-id (:id file)
|
||||
|
||||
deleted-at (ct/minus (ct/now) (ct/duration {:days 1}))]
|
||||
|
||||
(th/db-update! :profile
|
||||
{:deleted-at deleted-at}
|
||||
{:id (:id prof)})
|
||||
|
||||
(let [team-before (th/db-get :team {:id team-id} {::db/remove-deleted false})]
|
||||
(t/is (nil? (:deleted-at team-before))))
|
||||
|
||||
(let [result (th/run-task! :objects-gc {:min-age 0})]
|
||||
(t/is (pos? (:processed result))))
|
||||
|
||||
(let [profile-after (th/db-get :profile {:id (:id prof)} {::db/remove-deleted false})]
|
||||
(t/is (nil? profile-after)))
|
||||
|
||||
(let [team-after (th/db-get :team {:id team-id} {::db/remove-deleted false})]
|
||||
(t/is (nil? team-after)))
|
||||
|
||||
(let [project-after (th/db-get :project {:id project-id} {::db/remove-deleted false})]
|
||||
(t/is (nil? project-after)))
|
||||
|
||||
(let [file-after (th/db-get :file {:id file-id} {::db/remove-deleted false})]
|
||||
(t/is (nil? file-after)))))
|
||||
|
||||
(t/deftest email-blacklist-1
|
||||
(t/is (false? (email.blacklist/enabled? th/*system*)))
|
||||
|
||||
@@ -154,10 +154,14 @@
|
||||
(get-in % [:props :member-email])))
|
||||
events))]
|
||||
(doseq [event [create-organization update-organization]]
|
||||
(t/is (= (str (:id owner))
|
||||
(get-in event [:props :user-who-send-invitation])))
|
||||
(t/is (true? (get-in event [:props :team-belongs-to-organization])))
|
||||
(t/is (true? (get-in event [:props :adds-invitee-to-organization])))
|
||||
(t/is (true? (get-in event [:props :invitee-already-organization-member]))))
|
||||
|
||||
(t/is (= (str (:id owner))
|
||||
(get-in create-plain [:props :user-who-send-invitation])))
|
||||
(t/is (false? (get-in create-plain [:props :team-belongs-to-organization])))
|
||||
(t/is (false? (get-in create-plain [:props :adds-invitee-to-organization])))
|
||||
(t/is (false? (get-in create-plain [:props :invitee-already-organization-member])))))))
|
||||
@@ -521,6 +525,9 @@
|
||||
|
||||
(let [event (organization-event)]
|
||||
(t/is (= organization-id (get-in event [:props :organization-id])))
|
||||
(t/is (= (:id invitee) (get-in event [:props :user-id])))
|
||||
(t/is (= (:id inviter)
|
||||
(get-in event [:props :user-who-send-invitation])))
|
||||
(t/is (not (contains? (:props event) :organization-member-add-source)))
|
||||
(t/is (not (contains? (:props event) :belongs-to-team-on-add)))
|
||||
(t/is (not (contains? (:props event) :organization-member-count-before)))
|
||||
@@ -530,6 +537,10 @@
|
||||
(:origin @frontend-event)))
|
||||
(t/is (= organization-id
|
||||
(get-in @frontend-event [:props :organization-id])))
|
||||
(t/is (= (:id invitee)
|
||||
(get-in @frontend-event [:props :user-id])))
|
||||
(t/is (= (:id inviter)
|
||||
(get-in @frontend-event [:props :user-who-send-invitation])))
|
||||
(t/is (= "direct-organization-invitation"
|
||||
(get-in @frontend-event [:props :organization-member-add-source])))
|
||||
(t/is (false? (get-in @frontend-event [:props :belongs-to-team-on-add])))
|
||||
@@ -570,6 +581,9 @@
|
||||
(t/is (some #(= "accept-team-invitation-from" (:name %)) events))
|
||||
(t/is (= (:id team) (get-in event [:props :team-id])))
|
||||
(t/is (= organization-id (get-in event [:props :organization-id])))
|
||||
(t/is (= (:id invitee) (get-in event [:props :user-id])))
|
||||
(t/is (= (:id inviter)
|
||||
(get-in event [:props :user-who-send-invitation])))
|
||||
(t/is (not (contains? (:props event) :organization-member-add-source)))
|
||||
(t/is (not (contains? (:props event) :belongs-to-team-on-add)))
|
||||
(t/is (not (contains? (:props event) :organization-member-count-before)))
|
||||
@@ -578,6 +592,10 @@
|
||||
(t/is (= (:id team) (get-in @frontend-event [:props :team-id])))
|
||||
(t/is (= organization-id
|
||||
(get-in @frontend-event [:props :organization-id])))
|
||||
(t/is (= (:id invitee)
|
||||
(get-in @frontend-event [:props :user-id])))
|
||||
(t/is (= (:id inviter)
|
||||
(get-in @frontend-event [:props :user-who-send-invitation])))
|
||||
(t/is (= "team-invitation"
|
||||
(get-in @frontend-event [:props :organization-member-add-source])))
|
||||
(t/is (true? (get-in @frontend-event [:props :belongs-to-team-on-add])))
|
||||
@@ -1328,3 +1346,83 @@
|
||||
out (th/command! data)]
|
||||
(t/is (th/success? out))
|
||||
(t/is (= 1 (:call-count @mock)))))))
|
||||
|
||||
(t/deftest admin-cannot-remove-team-owner
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
admin (th/create-profile* 2 {:is-active true})
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})]
|
||||
|
||||
(th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id admin)
|
||||
:role :admin})
|
||||
|
||||
(let [out (th/command! {::th/type :delete-team-member
|
||||
::rpc/profile-id (:id admin)
|
||||
:team-id (:id team)
|
||||
:member-id (:id owner)})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (th/ex-of-type? (:error out) :validation))
|
||||
(t/is (th/ex-of-code? (:error out) :cant-remove-owner)))))
|
||||
|
||||
(t/deftest owner-can-remove-another-owner
|
||||
(let [owner1 (th/create-profile* 1 {:is-active true})
|
||||
owner2 (th/create-profile* 2 {:is-active true})
|
||||
team (th/create-team* 1 {:profile-id (:id owner1)})]
|
||||
|
||||
(th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id owner2)
|
||||
:role :owner})
|
||||
|
||||
(let [out (th/command! {::th/type :delete-team-member
|
||||
::rpc/profile-id (:id owner1)
|
||||
:team-id (:id team)
|
||||
:member-id (:id owner2)})]
|
||||
(t/is (th/success? out)))))
|
||||
|
||||
(t/deftest owner-can-remove-admin
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
admin (th/create-profile* 2 {:is-active true})
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})]
|
||||
|
||||
(th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id admin)
|
||||
:role :admin})
|
||||
|
||||
(let [out (th/command! {::th/type :delete-team-member
|
||||
::rpc/profile-id (:id owner)
|
||||
:team-id (:id team)
|
||||
:member-id (:id admin)})]
|
||||
(t/is (th/success? out)))))
|
||||
|
||||
(t/deftest admin-can-remove-admin
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
admin1 (th/create-profile* 2 {:is-active true})
|
||||
admin2 (th/create-profile* 3 {:is-active true})
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})]
|
||||
|
||||
(th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id admin1)
|
||||
:role :admin})
|
||||
|
||||
(th/create-team-role* {:team-id (:id team)
|
||||
:profile-id (:id admin2)
|
||||
:role :admin})
|
||||
|
||||
(let [out (th/command! {::th/type :delete-team-member
|
||||
::rpc/profile-id (:id admin1)
|
||||
:team-id (:id team)
|
||||
:member-id (:id admin2)})]
|
||||
(t/is (th/success? out)))))
|
||||
|
||||
(t/deftest delete-nonexistent-member-returns-not-found
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
team (th/create-team* 1 {:profile-id (:id owner)})
|
||||
fake-id (uuid/next)]
|
||||
|
||||
(let [out (th/command! {::th/type :delete-team-member
|
||||
::rpc/profile-id (:id owner)
|
||||
:team-id (:id team)
|
||||
:member-id fake-id})]
|
||||
(t/is (not (th/success? out)))
|
||||
(t/is (th/ex-of-type? (:error out) :not-found))
|
||||
(t/is (th/ex-of-code? (:error out) :member-does-not-exist)))))
|
||||
@@ -128,3 +128,80 @@
|
||||
(let [result (:result out)]
|
||||
(t/is (contains? result :file))
|
||||
(t/is (contains? result :project)))))))
|
||||
|
||||
(t/deftest share-link-token-disclosure
|
||||
(let [owner (th/create-profile* 1 {:is-active true})
|
||||
proj-id (:default-project-id owner)
|
||||
|
||||
file (th/create-file* 1 {:profile-id (:id owner)
|
||||
:project-id proj-id
|
||||
:is-shared false})
|
||||
|
||||
page-a (get-in file [:data :pages 0])
|
||||
page-b (uuid/random)
|
||||
|
||||
;; Add a second page to the file
|
||||
_ (th/command! {::th/type :update-file
|
||||
::rpc/profile-id (:id owner)
|
||||
:id (:id file)
|
||||
:session-id (uuid/random)
|
||||
:revn 0
|
||||
:vern 0
|
||||
:changes [{:type :add-page
|
||||
:id page-b
|
||||
:page {:id page-b
|
||||
:name "Page B"
|
||||
:options {}
|
||||
:objects {}}}]})
|
||||
|
||||
;; Create Link A: restrictive (no pages, team-only comments/inspect)
|
||||
link-a (th/command! {::th/type :create-share-link
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:pages #{}
|
||||
:who-comment "team"
|
||||
:who-inspect "team"})
|
||||
link-a-id (get-in link-a [:result :id])
|
||||
|
||||
;; Create Link B: permissive (all pages, all can comment/inspect)
|
||||
link-b (th/command! {::th/type :create-share-link
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)
|
||||
:pages #{page-a page-b}
|
||||
:who-comment "all"
|
||||
:who-inspect "all"})
|
||||
link-b-id (get-in link-b [:result :id])]
|
||||
|
||||
(t/testing "restrictive share-link holder cannot see other share-link tokens"
|
||||
(let [out (th/command! {::th/type :get-view-only-bundle
|
||||
:share-id link-a-id
|
||||
:file-id (:id file)})
|
||||
err (:error out)
|
||||
result (:result out)
|
||||
share-links (:share-links result)]
|
||||
|
||||
;; Should not error
|
||||
(t/is (nil? err))
|
||||
|
||||
;; Should only see the share-link used for authentication
|
||||
(t/is (= 1 (count share-links)))
|
||||
(t/is (= link-a-id (:id (first share-links))))
|
||||
|
||||
;; Should NOT see Link B's token
|
||||
(t/is (not (some #(= link-b-id (:id %)) share-links)))))
|
||||
|
||||
(t/testing "team member still sees all share-links"
|
||||
(let [out (th/command! {::th/type :get-view-only-bundle
|
||||
::rpc/profile-id (:id owner)
|
||||
:file-id (:id file)})
|
||||
err (:error out)
|
||||
result (:result out)
|
||||
share-links (:share-links result)]
|
||||
|
||||
;; Should not error
|
||||
(t/is (nil? err))
|
||||
|
||||
;; Team member should see both share-links
|
||||
(t/is (= 2 (count share-links)))
|
||||
(t/is (some #(= link-a-id (:id %)) share-links))
|
||||
(t/is (some #(= link-b-id (:id %)) share-links))))))
|
||||
@@ -80,6 +80,24 @@
|
||||
(t/is (false? (ssrf/safe-url? "http://[fd00::1]/foo")))
|
||||
(t/is (false? (ssrf/safe-url? "http://[fc00::1]/foo"))))
|
||||
|
||||
(t/deftest validate-url-blocks-nat64-encoded-metadata
|
||||
;; 64:ff9b::a9fe:a9fe embeds 169.254.169.254 (cloud metadata)
|
||||
(t/is (false? (ssrf/safe-url? "http://[64:ff9b::a9fe:a9fe]/latest/meta-data/"))))
|
||||
|
||||
(t/deftest validate-url-blocks-nat64-encoded-loopback
|
||||
;; 64:ff9b::7f00:0001 embeds 127.0.0.1
|
||||
(t/is (false? (ssrf/safe-url? "http://[64:ff9b::7f00:1]/foo"))))
|
||||
|
||||
(t/deftest validate-url-blocks-6to4-encoded-private
|
||||
;; 2002:a00:1:: embeds 10.0.0.1; 2002:c0a8:101:: embeds 192.168.1.1
|
||||
(t/is (false? (ssrf/safe-url? "http://[2002:a00:1::1]/foo")))
|
||||
(t/is (false? (ssrf/safe-url? "http://[2002:c0a8:101::1]/foo"))))
|
||||
|
||||
(t/deftest validate-url-blocks-teredo-encoded-addresses
|
||||
;; Teredo server prefix 2001:0000::/32
|
||||
(t/is (false? (ssrf/safe-url?
|
||||
"http://[2001:0000:4136:e378:8000:63bf:3fff:fdd2]/foo"))))
|
||||
|
||||
(t/deftest validate-url-blocks-encoded-loopback
|
||||
;; Decimal encoding of 127.0.0.1 = 2130706433
|
||||
;; InetAddress normalizes this to 127.0.0.1
|
||||
|
||||
+4
-4
@@ -17,14 +17,14 @@
|
||||
org.slf4j/slf4j-api {:mvn/version "2.0.18"}
|
||||
pl.tkowalcz.tjahzi/log4j2-appender {:mvn/version "0.9.43"}
|
||||
|
||||
selmer/selmer {:mvn/version "1.13.4"}
|
||||
selmer/selmer {:mvn/version "1.13.5"}
|
||||
criterium/criterium {:mvn/version "0.4.6"}
|
||||
|
||||
metosin/jsonista {:mvn/version "1.0.0"
|
||||
:exclusions [com.fasterxml.jackson.core/jackson-core
|
||||
com.fasterxml.jackson.core/jackson-databind]}
|
||||
com.fasterxml.jackson.core/jackson-core {:mvn/version "2.22.1"}
|
||||
com.fasterxml.jackson.core/jackson-databind {:mvn/version "2.22.1"}
|
||||
com.fasterxml.jackson.core/jackson-core {:mvn/version "2.22.2"}
|
||||
com.fasterxml.jackson.core/jackson-databind {:mvn/version "2.22.2"}
|
||||
|
||||
|
||||
metosin/malli {:mvn/version "0.20.1"}
|
||||
@@ -60,7 +60,7 @@
|
||||
{:dev
|
||||
{:extra-deps
|
||||
{org.clojure/tools.namespace {:mvn/version "1.5.1"}
|
||||
thheller/shadow-cljs {:mvn/version "3.4.11"}
|
||||
thheller/shadow-cljs {:mvn/version "3.5.0"}
|
||||
com.clojure-goes-fast/clj-async-profiler {:mvn/version "2.0.0-beta1"}
|
||||
com.bhauman/rebel-readline {:mvn/version "0.1.11"}
|
||||
criterium/criterium {:mvn/version "0.4.6"}
|
||||
|
||||
+3
-3
@@ -4,18 +4,18 @@
|
||||
"license": "MPL-2.0",
|
||||
"author": "Kaleidos INC Sucursal en España SL",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@11.20.0+sha512.9a6f330a95b66446ea088faf1521405a8a01f07fde7124cc9958dfed52d4bb436737e65b08f85f37b46fcba375092558ac51262b816844b22f63406ed166bfee",
|
||||
"packageManager": "pnpm@12.0.0+sha512.9e2e3dc3911995868dc94b8175c217c27e95408fa03b4a22749778f2b34f773b77cdd3b39ede8171b22fcd53be6a35342e9fac9948a68ef58df6488ce89a7e67",
|
||||
"type": "module",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/penpot/penpot"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^10.0.4",
|
||||
"concurrently": "^10.0.5",
|
||||
"nodemon": "^3.1.14",
|
||||
"prettier": "3.9.6",
|
||||
"source-map-support": "^0.5.21",
|
||||
"ws": "^8.21.2"
|
||||
"ws": "^8.21.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"date-fns": "^4.4.0"
|
||||
|
||||
Generated
+111
-10
@@ -1,3 +1,104 @@
|
||||
---
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
configDependencies: {}
|
||||
packageManagerDependencies:
|
||||
pnpm:
|
||||
specifier: 12.0.0
|
||||
version: 12.0.0
|
||||
|
||||
packages:
|
||||
|
||||
'@pnpm/exe.darwin-arm64@12.0.0':
|
||||
resolution: {integrity: sha512-sqeoPfVMIfQhbwzDrKraXY2ynyuWClFqzvfImzAS/yczEru1m5SGvQ9kgFPDvQzJZ9AetedgJeDZC6qYvH8/tQ==}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
'@pnpm/exe.darwin-x64@12.0.0':
|
||||
resolution: {integrity: sha512-Quc3J6c9cGTy+LDgz1cLVgCNOU9IERuyAlDoEj0DCilKqvo50Jx1GV8k74iwn4J9fFSKkm8JrwNvtTDj3uWnUA==}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
'@pnpm/exe.linux-arm64-musl@12.0.0':
|
||||
resolution: {integrity: sha512-EVWd3OTmgsMFhXx69b5JxIzoabG9Ma7m4OeTaf0ZKBzMnfYi8u21NDQo92ToMrdYL5dYDDCHsyYIjXzk+d0HhA==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@pnpm/exe.linux-arm64@12.0.0':
|
||||
resolution: {integrity: sha512-cXHHW8M4rAPsYNkKZO9WVcpLLK55i9EaIsZPfIqUuY2eopd5LqnFyBge54HCh1GC0yCX8ySn0hYIi+4OyAEoDg==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@pnpm/exe.linux-x64-musl@12.0.0':
|
||||
resolution: {integrity: sha512-UcXwMdFjly0mpddkGigHKTxe27IMv2fUK4IWW/MHmJ3yMguxXmkwNlEI4aE+G1HO2TLo20uNEUWD4ymLe/DaCQ==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [musl]
|
||||
|
||||
'@pnpm/exe.linux-x64@12.0.0':
|
||||
resolution: {integrity: sha512-6Rsl+zEWMOmus7v7/9J3OE8EMvHyNAfxYmDfmhQG4J0985OuT3G3Ho9NSGHjkBn4aU4bgklWifRhe1HX8dUSyw==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
libc: [glibc]
|
||||
|
||||
'@pnpm/exe.win32-arm64@12.0.0':
|
||||
resolution: {integrity: sha512-O5F76A4oVFrpDGdFxEszRIThOSBfjHdH5c006gR+7UTCfiXrukr1XfqPungUI1DXcSR5gb9jBsPQqQZOAoOOxw==}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
|
||||
'@pnpm/exe.win32-x64@12.0.0':
|
||||
resolution: {integrity: sha512-5dKFajIEWJ1ai+KHXFJvskY6vchbunmLwSUV2ywbLymcmJjfY5XJVpgzPCyIoVCMVG0zHorr66+hM8h8b3aRfQ==}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
pnpm@12.0.0:
|
||||
resolution: {integrity: sha512-ni49w5EZlYaNyUuBdcIXwn6VQI+gO0oidJd48rNPdzt3zdOznt6BcbIvzVO+ajU0Lp+smUimjvWN9kiM6Jp+Zw==}
|
||||
engines: {node: '>=18.*'}
|
||||
hasBin: true
|
||||
|
||||
snapshots:
|
||||
|
||||
'@pnpm/exe.darwin-arm64@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.darwin-x64@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.linux-arm64-musl@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.linux-arm64@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.linux-x64-musl@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.linux-x64@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.win32-arm64@12.0.0':
|
||||
optional: true
|
||||
|
||||
'@pnpm/exe.win32-x64@12.0.0':
|
||||
optional: true
|
||||
|
||||
pnpm@12.0.0:
|
||||
optionalDependencies:
|
||||
'@pnpm/exe.darwin-arm64': 12.0.0
|
||||
'@pnpm/exe.darwin-x64': 12.0.0
|
||||
'@pnpm/exe.linux-arm64': 12.0.0
|
||||
'@pnpm/exe.linux-arm64-musl': 12.0.0
|
||||
'@pnpm/exe.linux-x64': 12.0.0
|
||||
'@pnpm/exe.linux-x64-musl': 12.0.0
|
||||
'@pnpm/exe.win32-arm64': 12.0.0
|
||||
'@pnpm/exe.win32-x64': 12.0.0
|
||||
|
||||
---
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
@@ -13,8 +114,8 @@ importers:
|
||||
version: 4.4.0
|
||||
devDependencies:
|
||||
concurrently:
|
||||
specifier: ^10.0.4
|
||||
version: 10.0.4
|
||||
specifier: ^10.0.5
|
||||
version: 10.0.5
|
||||
nodemon:
|
||||
specifier: ^3.1.14
|
||||
version: 3.1.14
|
||||
@@ -25,8 +126,8 @@ importers:
|
||||
specifier: ^0.5.21
|
||||
version: 0.5.21
|
||||
ws:
|
||||
specifier: ^8.21.2
|
||||
version: 8.21.2
|
||||
specifier: ^8.21.3
|
||||
version: 8.21.3
|
||||
|
||||
packages:
|
||||
|
||||
@@ -73,8 +174,8 @@ packages:
|
||||
resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
concurrently@10.0.4:
|
||||
resolution: {integrity: sha512-trZql+7l/0+WRAsAnEdctr4+iiOS6ZrViI6H8QWcCF9MFS/LT0dKpe8vluB1to6it+OxSI4VospFTIFMW8DJRw==}
|
||||
concurrently@10.0.5:
|
||||
resolution: {integrity: sha512-JaP/CoftUrCcAFW/g//RbgEGwlelnEae6cfBLgH6ZdO6s8jPkn6p9SB9u6pdVxYXoiSnFqseOlHfrEfF82TVOg==}
|
||||
engines: {node: '>=22'}
|
||||
hasBin: true
|
||||
|
||||
@@ -234,8 +335,8 @@ packages:
|
||||
resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
ws@8.21.2:
|
||||
resolution: {integrity: sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==}
|
||||
ws@8.21.3:
|
||||
resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
peerDependencies:
|
||||
bufferutil: ^4.0.1
|
||||
@@ -303,7 +404,7 @@ snapshots:
|
||||
strip-ansi: 7.2.0
|
||||
wrap-ansi: 9.0.2
|
||||
|
||||
concurrently@10.0.4:
|
||||
concurrently@10.0.5:
|
||||
dependencies:
|
||||
chalk: 5.6.2
|
||||
rxjs: 7.8.2
|
||||
@@ -439,7 +540,7 @@ snapshots:
|
||||
string-width: 7.2.0
|
||||
strip-ansi: 7.2.0
|
||||
|
||||
ws@8.21.2: {}
|
||||
ws@8.21.3: {}
|
||||
|
||||
y18n@5.0.8: {}
|
||||
|
||||
|
||||
@@ -134,7 +134,6 @@
|
||||
:feature-text-editor-v2-html-paste "text-editor/v2-html-paste"
|
||||
:feature-text-editor-wasm "text-editor-wasm/v1"
|
||||
:feature-render-wasm "render-wasm/v1"
|
||||
:feature-wasm-export "wasm-export/v1"
|
||||
:feature-variants "variants/v1"
|
||||
:feature-token-input "tokens/numeric-input"
|
||||
nil))
|
||||
|
||||
Loaded 100 of 402 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user