Compare commits

..

21 Commits

Author SHA1 Message Date
Aitor Moreno
dbbf3ab41d WIP 2025-07-24 10:16:48 +02:00
Aitor Moreno
5b26a9a269 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
f56420d327 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
462ac77fcd WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
fd3c3d720e WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
a20e0c5733 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
26ca7ed566 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
2a58387b02 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
84ad14183e WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
0704fa5df6 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
b438f4dfa1 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
4a85a9ac46 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
a77ca32bf8 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
2a7ab93892 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
03e4ff828b WIP 2025-07-24 09:23:19 +02:00
Alejandro Alonso
2ab5487a84 WIP fix partial 2025-07-24 09:23:19 +02:00
Aitor Moreno
f60254e3eb WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
4274db11e3 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
ee3bf72034 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
446fc53f00 WIP 2025-07-24 09:23:19 +02:00
Aitor Moreno
3af70a965d WIP 2025-07-24 09:23:19 +02:00
810 changed files with 37043 additions and 71171 deletions

View File

@@ -45,16 +45,10 @@
:potok/reify-type
{:level :error}
:missing-protocol-method
{:level :off}
:unresolved-namespace
{:level :warning
:exclude [data_readers]}
:unused-value
{:level :off}
:single-key-in
{:level :warning}
@@ -70,9 +64,6 @@
:redundant-nested-call
{:level :off}
:redundant-str-call
{:level :off}
:earmuffed-var-not-dynamic
{:level :off}

View File

@@ -1,97 +0,0 @@
name: Build and Upload Penpot Bundle
on:
# Create bundle from manual action
workflow_dispatch:
inputs:
gh_ref:
description: 'Name of the branch'
type: string
required: true
default: 'develop'
build_wasm:
description: 'BUILD_WASM. Valid values: yes, no'
type: string
required: false
default: 'yes'
build_storybook:
description: 'BUILD_STORYBOOK. Valid values: yes, no'
type: string
required: false
default: 'yes'
workflow_call:
inputs:
gh_ref:
description: 'Name of the branch'
type: string
required: true
default: 'develop'
build_wasm:
description: 'BUILD_WASM. Valid values: yes, no'
type: string
required: false
default: 'yes'
build_storybook:
description: 'BUILD_STORYBOOK. Valid values: yes, no'
type: string
required: false
default: 'yes'
jobs:
build-bundle:
name: Build and Upload Penpot Bundle
runs-on: ubuntu-24.04
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.gh_ref }}
- name: Extract some useful variables
id: vars
run: |
echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT
- name: Run manage.sh build-bundle from host
env:
BUILD_WASM: ${{ inputs.build_wasm }}
BUILD_STORYBOOK: ${{ inputs.build_storybook }}
run: ./manage.sh build-bundle
- name: Prepare directories for zipping
run: |
mkdir zips
mv bundles penpot
- name: Create zip bundle
run: |
echo "📦 Packaging Penpot bundle..."
zip -r zips/penpot.zip penpot
- name: Upload Penpot bundle to S3
if: github.ref_type == 'branch'
run: |
aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}-latest.zip
aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.commit_hash }}.zip
- name: Upload Penpot bundle to S3
if: github.ref_type == 'tag'
run: |
aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip
- name: Notify Mattermost
if: failure()
uses: mattermost/action-mattermost-notify@master
with:
MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
TEXT: |
❌ *[PENPOT] Error during the execution of the job*
📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}`
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

129
.github/workflows/build-bundles.yml vendored Normal file
View File

@@ -0,0 +1,129 @@
name: Build and Upload Penpot Bundles non-prod
on:
# Create bundler for every tag
push:
tags:
- '**' # Pattern matched against refs/tags
# Create bundler every hour between 5:00 and 20:00 on working days
schedule:
- cron: '0 5-20 * * 1-5'
# Create bundler from manual action
workflow_dispatch:
inputs:
zip_mode:
# zip_mode defines how the build artifacts are packaged:
# - 'individual': creates one ZIP file per component (frontend, backend, exporter)
# - 'all': creates a single ZIP containing all components
# - null: for the rest of cases (non-manual events)
description: 'Bundle packaging mode'
required: false
default: 'individual'
type: choice
options:
- individual
- all
jobs:
build-bundles:
name: Build and Upload Penpot Bundles
runs-on: ubuntu-24.04
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract somer useful variables
id: vars
run: |
echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
echo "gh_branch=${{ github.base_ref || github.ref_name }}" >> $GITHUB_OUTPUT
# Set up Docker Buildx for multi-arch build
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Run manage.sh build-bundle from host
run: ./manage.sh build-bundle
- name: Prepare directories for zipping
run: |
mkdir zips
mv bundles penpot
- name: Create zip bundles for zip_mode == 'all'
if: ${{ github.event.inputs.zip_mode == 'all' }}
run: |
echo "📦 Packaging Penpot 'all' bundles..."
zip -r zips/penpot-all-bundles.zip penpot
- name: Create zip bundles for zip_mode != 'all'
if: ${{ github.event.inputs.zip_mode != 'all' }}
run: |
echo "📦 Packaging Penpot 'individual' bundles..."
zip -r zips/penpot-frontend.zip penpot/frontend
zip -r zips/penpot-backend.zip penpot/backend
zip -r zips/penpot-exporter.zip penpot/exporter
- name: Upload unified 'all' bundle
if: ${{ github.event.inputs.zip_mode == 'all' }}
uses: actions/upload-artifact@v4
with:
name: penpot-all-bundles
path: zips/penpot-all-bundles.zip
- name: Upload individual bundles
if: ${{ github.event.inputs.zip_mode != 'all' }}
uses: actions/upload-artifact@v4
with:
name: penpot-individual-bundles
path: |
zips/penpot-frontend.zip
zips/penpot-backend.zip
zips/penpot-exporter.zip
- name: Upload unified 'all' bundle to S3
if: ${{ github.event.inputs.zip_mode == 'all' }}
run: |
aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.gh_branch}}.zip
aws s3 cp zips/penpot-all-bundles.zip s3://${{ secrets.S3_BUCKET }}/penpot-all-bundles-${{ steps.vars.outputs.commit_hash }}.zip
- name: Upload 'individual' bundles to S3
if: ${{ github.event.inputs.zip_mode != 'all' }}
run: |
for name in penpot-frontend penpot-backend penpot-exporter; do
aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.gh_branch }}-latest.zip
aws s3 cp zips/${name}.zip s3://${{ secrets.S3_BUCKET }}/${name}-${{ steps.vars.outputs.commit_hash }}.zip
done
- name: Notify Mattermost about automatic bundles
if: github.event_name == 'pull_request'
uses: mattermost/action-mattermost-notify@master
with:
MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
TEXT: |
📦 *Penpot bundle automatically generated*
📄 PR: ${{ github.event.pull_request.title }}
🔁 From: \`${{ github.head_ref }}\` to \`{{ github.base_ref }}\`
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Notify Mattermost about manual bundles
if: github.event_name == 'workflow_dispatch'
uses: mattermost/action-mattermost-notify@master
with:
MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
TEXT: |
📦 *Penpot bundle manually generated*
📄 Triggered from branch: `${{ github.ref_name}}`
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Print artifact summary URL
run: |
echo "📦 Artifacts available at:"
echo "🔗 https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

View File

@@ -1,14 +0,0 @@
name: DEVELOP - Build and Upload Penpot Bundle
on:
schedule:
- cron: '16 5-20 * * 1-5'
jobs:
build-develop-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
with:
gh_ref: "develop"
build_wasm: "yes"
build_storybook: "yes"

View File

@@ -1,14 +0,0 @@
name: STAGING - Build and Upload Penpot Bundle
on:
schedule:
- cron: '36 5-20 * * 1-5'
jobs:
build-staging-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
with:
gh_ref: "staging"
build_wasm: "yes"
build_storybook: "yes"

View File

@@ -1,15 +0,0 @@
name: TAG - Build and Upload Penpot Bundle
on:
push:
tags:
- '*'
jobs:
build-tag-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
with:
gh_ref: ${{ github.ref_name }}
build_wasm: "no"
build_storybook: "yes"

View File

@@ -26,7 +26,7 @@ jobs:
- name: Check Commit Type
uses: gsactions/commit-message-checker@v2
with:
pattern: '^(Merge|Revert|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind):)\s[A-Z].*[^.]$'
pattern: '^(Merge|:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle):)\s[A-Z].*[^.]$'
flags: 'gm'
error: 'Commit should match CONTRIBUTING.md guideline'
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request

1
.gitignore vendored
View File

@@ -31,7 +31,6 @@
/.clj-kondo/.cache
/_dump
/notes
/playground/
/backend/*.md
/backend/*.sql
/backend/*.txt

2
.nvmrc
View File

@@ -1 +1 @@
v22.19.0
v22.13.1

View File

@@ -1,83 +1,19 @@
# CHANGELOG
## 2.10.1
### :sparkles: New features & Enhancements
- Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366)
### :bug: Bugs fixed
- Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244)
## 2.10.0
## 2.10.0 (Unreleased)
### :rocket: Epics and highlights
- Variants
### :boom: Breaking changes & Deprecations
### :heart: Community contributions (Thank you!)
### :sparkles: New features & Enhancements
- Add efficiency enhancements to right sidebar [Github #7182](https://github.com/penpot/penpot/pull/7182)
- Add defaults for artboard drawing [Taiga #494](https://tree.taiga.io/project/penpot/us/494?milestone=465047)
- Continuous display of distances between elements when moving a layer with the keyboard [Taiga #1780](https://tree.taiga.io/project/penpot/us/1780)
- New Number token - unitless values [Taiga #10936](https://tree.taiga.io/project/penpot/us/10936)
- New font-family token [Taiga #10937](https://tree.taiga.io/project/penpot/us/10937)
- New text case token [Taiga #10942](https://tree.taiga.io/project/penpot/us/10942)
- New text-decoration token [Taiga #10941](https://tree.taiga.io/project/penpot/us/10941)
- New letter spacing token [Taiga #10940](https://tree.taiga.io/project/penpot/us/10940)
- New font weight token [Taiga #10939](https://tree.taiga.io/project/penpot/us/10939)
- Upgrade Node to v22.18.0 [Github #7283](https://github.com/penpot/penpot/pull/7283)
- Upgrade the base docker image for penpot frontend to v1.29.1 [Github #7283](https://github.com/penpot/penpot/pull/7283)
- Create variant from an existing component [Taiga #2088](https://tree.taiga.io/project/penpot/us/2088)
- Create variant from an existing variant [Taiga #8282](https://tree.taiga.io/project/penpot/us/8282)
- Actions over a component with variants [Taiga #10503](https://tree.taiga.io/project/penpot/us/10503)
- Create a variant by dragging a component into a component with variants [Taiga #8134](https://tree.taiga.io/project/penpot/us/8134)
- Transform a variant into an individual component [Taiga #8141](https://tree.taiga.io/project/penpot/us/8141)
- Delete variant [Taiga #6890](https://tree.taiga.io/project/penpot/us/6890)
- Restore an orphaned copy of a variant [Taiga #10446](https://tree.taiga.io/project/penpot/us/10446)
- Add, Edit & Delete variant properties name and value [Taiga #6892](https://tree.taiga.io/project/penpot/us/6892)
- Retrieve variants [Taiga #6888](https://tree.taiga.io/project/penpot/us/6888)
- Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
- Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
- Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
### :bug: Bugs fixed
- Display strokes information in inspect tab [Taiga #11154](https://tree.taiga.io/project/penpot/issue/11154)
- Fix problem with booleans selection [Taiga #11627](https://tree.taiga.io/project/penpot/issue/11627)
- Fix missing font when copy&paste a chunk of text [Taiga #11522](https://tree.taiga.io/project/penpot/issue/11522)
- Fix bad swap slot after two swaps [Taiga #11659](https://tree.taiga.io/project/penpot/issue/11659)
- Fix missing package for the `penpot_exporter` Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
- Fix issue where multiple dropdown menus could be opened simultaneously on the dashboard page [Taiga #11500](https://tree.taiga.io/project/penpot/issue/11500)
- Fix font size/variant not updated when editing a text [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11552)
- Fix issue where Alt + arrow keys shortcut interferes with letter-spacing when moving text layers [Taiga #11552](https://tree.taiga.io/project/penpot/issue/11771)
- Fix consistency issues on how font variants are visualized [Taiga #11499](https://tree.taiga.io/project/penpot/us/11499)
- Fix parsing rx and ry SVG values for rect radius [Taiga #11861](https://tree.taiga.io/project/penpot/issue/11861)
- Misleading affordance in saved versions [Taiga #11887](https://tree.taiga.io/project/penpot/issue/11887)
- Fix pasting RTF text crashes penpot [Taiga #11717](https://tree.taiga.io/project/penpot/issue/11717)
- Fix navigation arrows in Libraries & Templates carousel [Taiga #10609](https://tree.taiga.io/project/penpot/issue/10609)
- Fix applying tokens with zero value to size [Taiga #11618](https://tree.taiga.io/project/penpot/issue/11618)
- Fix typo [Taiga #11969](https://tree.taiga.io/project/penpot/issue/11969)
- Fix typo [Taiga #11970](https://tree.taiga.io/project/penpot/issue/11970)
- Fix typos [Taiga #11971](https://tree.taiga.io/project/penpot/issue/11971)
- Fix inconsistent naming for "Flatten" [Taiga #8371](https://tree.taiga.io/project/penpot/issue/8371)
- Layout item tokens should be unapplied when moving out of a layout [Taiga #11012](https://tree.taiga.io/project/penpot/issue/11012)
- Fix incorrect date displayed for support plan [Taiga #11986](https://tree.taiga.io/project/penpot/issue/11986)
- Fix can't import 'borderWidth' type token [#132](https://github.com/tokens-studio/penpot/issues/132)
- Fix moving elements up or down while pressing alt [Taiga Issue #11992](https://tree.taiga.io/project/penpot/issue/11992)
- Fix conflicting shortcuts (remove dec/inc line height and letter spacing) [Taiga #12102](https://tree.taiga.io/project/penpot/issue/12102)
- Fix conflicting shortcuts (remove text-align shortcuts) [Taiga #12047](https://tree.taiga.io/project/penpot/issue/12047)
- Fix export file with empty tokens library [Taiga #12137](https://tree.taiga.io/project/penpot/issue/12137)
- Fix context menu on spacing tokens [Taiga #12141](https://tree.taiga.io/project/penpot/issue/12141)
## 2.9.0
## 2.9.0 (Unreleased)
### :rocket: Epics and highlights
@@ -102,11 +38,9 @@
- Hide bounding box while editing visual effects [Taiga #11576](https://tree.taiga.io/project/penpot/issue/11576)
- Improved text layer resizing: Allow double-click on text bounding box to set auto-width/auto-height [Taiga #11577](https://tree.taiga.io/project/penpot/issue/11577)
- Improve text layer auto-resize: auto-width switches to auto-height on horizontal resize, and only switches to fixed on vertical resize [Taiga #11578](https://tree.taiga.io/project/penpot/issue/11578)
- Highlight first font in font selector search. Apply only on Enter or click. [Taiga #11579](https://tree.taiga.io/project/penpot/issue/11579)
- Add the ability to show login dialog on profile settings [Github #6871](https://github.com/penpot/penpot/pull/6871)
- Improve the application of tokens with object specific tokens [Taiga #10209](https://tree.taiga.io/project/penpot/us/10209)
- Add info to apply-token event [Taiga #11710](https://tree.taiga.io/project/penpot/task/11710)
- Fix double click on set name input [Taiga #11747](https://tree.taiga.io/project/penpot/issue/11747)
### :bug: Bugs fixed
@@ -120,32 +54,18 @@
- Keep color data when copying from info tab into CSS [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
- Update HSL values to modern syntax as defined in W3C CSS Color Module Level 4 [Taiga #11144](https://tree.taiga.io/project/penpot/issue/11144)
- Fix main component receives focus and is selected when using 'Show Main Component' [Taiga #11402](https://tree.taiga.io/project/penpot/issue/11402)
- Fix UI theme selection from main menu [Taiga #11567](https://tree.taiga.io/project/penpot/issue/11567)
- Fix duplicating pages with mainInstance shapes nested inside groups [Taiga #10774](https://tree.taiga.io/project/penpot/issue/10774)
- Fix ESC key not closing Add/Manage Libraries modal [Taiga #11523](https://tree.taiga.io/project/penpot/issue/11523)
- Fix copying a shadow color from info tab [Taiga #11211](https://tree.taiga.io/project/penpot/issue/11211)
- Fix remove color button in the gradient editor [Taiga #11623](https://tree.taiga.io/project/penpot/issue/11623)
- Fix "Copy as SVG" generates different code from the Inspect panel [Taiga #11519](https://tree.taiga.io/project/penpot/issue/11519)
- Fix overriden tokens in text copies are not preserved [Taiga #11486](https://tree.taiga.io/project/penpot/issue/11486)
- Fix problem when changing between flex/grid layout [Taiga #11625](https://tree.taiga.io/project/penpot/issue/11625)
- Fix opacity on stroke gradients [Taiga #11646](https://tree.taiga.io/project/penpot/issue/11646)
- Fix change from gradient to solid color [Taiga #11648](https://tree.taiga.io/project/penpot/issue/11648)
- Fix the context menu always closes after any action [Taiga #11624](https://tree.taiga.io/project/penpot/issue/11624)
- Fix X & Y position do not sincronize with tokens [Taiga #11617](https://tree.taiga.io/project/penpot/issue/11617)
- Fix tooltip position after first time [Taiga #11688](https://tree.taiga.io/project/penpot/issue/11688)
- Fix inconsistent ordering of pinned projects on dashboard sidebar [Taiga #11674](https://tree.taiga.io/project/penpot/issue/11674)
- Fix export button width on inspect tab [Taiga #11394](https://tree.taiga.io/project/penpot/issue/11394)
- Fix stroke width token application [Taiga #11724](https://tree.taiga.io/project/penpot/issue/11724)
- Fix number token application on shape [Taiga #11331](https://tree.taiga.io/project/penpot/task/11331)
- Fix auto height is fixed in the HTML inspect tab for text elements [Taiga #11680](https://tree.taiga.io/project/penpot/task/11680)
## 2.8.1
## 2.8.1 (Unreleased)
### :bug: Bugs fixed
- Fix unexpected exception on processing old texts [Github #6889](https://github.com/penpot/penpot/pull/6889)
- Fix error on inspect tab when selecting multiple shapes [Taiga #11655](https://tree.taiga.io/project/penpot/issue/11655)
- Fix missing package for the penport_exporter Docker image [GitHub #7205](https://github.com/penpot/penpot/issues/7025)
## 2.8.0

View File

@@ -3,7 +3,7 @@
:deps
{penpot/common {:local/root "../common"}
org.clojure/clojure {:mvn/version "1.12.2"}
org.clojure/clojure {:mvn/version "1.12.1"}
org.clojure/tools.namespace {:mvn/version "1.5.0"}
com.github.luben/zstd-jni {:mvn/version "1.5.7-3"}
@@ -38,7 +38,7 @@
metosin/reitit-core {:mvn/version "0.9.1"}
nrepl/nrepl {:mvn/version "1.3.1"}
org.postgresql/postgresql {:mvn/version "42.7.7"}
org.postgresql/postgresql {:mvn/version "42.7.6"}
org.xerial/sqlite-jdbc {:mvn/version "3.49.1.0"}
com.zaxxer/HikariCP {:mvn/version "6.3.0"}
@@ -65,7 +65,7 @@
;; Pretty Print specs
pretty-spec/pretty-spec {:mvn/version "0.1.4"}
software.amazon.awssdk/s3 {:mvn/version "2.33.8"}}
software.amazon.awssdk/s3 {:mvn/version "2.31.55"}}
:paths ["src" "resources" "target/classes"]
:aliases

View File

@@ -6,14 +6,12 @@
(ns user
(:require
[app.binfile.common :as bfc]
[app.common.data :as d]
[app.common.debug :as debug]
[app.common.exceptions :as ex]
[app.common.files.helpers :as cfh]
[app.common.fressian :as fres]
[app.common.geom.matrix :as gmt]
[app.common.json :as json]
[app.common.logging :as l]
[app.common.perf :as perf]
[app.common.pprint :as pp]
@@ -21,9 +19,8 @@
[app.common.schema.desc-js-like :as smdj]
[app.common.schema.desc-native :as smdn]
[app.common.schema.generators :as sg]
[app.common.schema.openapi :as oapi]
[app.common.spec :as us]
[app.common.time :as ct]
[app.common.json :as json]
[app.common.transit :as t]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
@@ -33,9 +30,9 @@
[app.srepl.helpers :as srepl.helpers]
[app.srepl.main :as srepl]
[app.util.blob :as blob]
[app.util.time :as dt]
[clj-async-profiler.core :as prof]
[clojure.contrib.humanize :as hum]
[clojure.datafy :refer [datafy]]
[clojure.java.io :as io]
[clojure.pprint :refer [pprint print-table]]
[clojure.repl :refer :all]

View File

@@ -12,7 +12,7 @@ export PENPOT_FLAGS="\
enable-login-with-gitlab \
enable-backend-worker \
enable-backend-asserts \
disable-feature-fdata-pointer-map \
enable-feature-fdata-pointer-map \
enable-feature-fdata-objects-map \
enable-audit-log \
enable-transit-readable-response \
@@ -28,11 +28,11 @@ export PENPOT_FLAGS="\
enable-auto-file-snapshot \
enable-webhooks \
enable-access-tokens \
disable-tiered-file-data-storage \
enable-tiered-file-data-storage \
enable-file-validation \
enable-file-schema-validation \
enable-subscriptions \
disable-subscriptions-old";
enable-subscriptions-old";
# Default deletion delay for devenv
export PENPOT_DELETION_DELAY="24h"

View File

@@ -13,7 +13,7 @@ export PENPOT_FLAGS="\
enable-login-with-ldap \
enable-transit-readable-response \
enable-demo-users \
disable-feature-fdata-pointer-map \
enable-feature-fdata-pointer-map \
enable-feature-fdata-objects-map \
disable-secure-session-cookies \
enable-rpc-climit \
@@ -21,11 +21,11 @@ export PENPOT_FLAGS="\
enable-quotes \
enable-file-snapshot \
enable-access-tokens \
disable-tiered-file-data-storage \
enable-tiered-file-data-storage \
enable-file-validation \
enable-file-schema-validation \
enable-subscriptions \
disable-subscriptions-old";
enable-subscriptions-old ";
# Default deletion delay for devenv
export PENPOT_DELETION_DELAY="24h"

View File

@@ -13,7 +13,6 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
@@ -29,6 +28,7 @@
[app.tokens :as tokens]
[app.util.inet :as inet]
[app.util.json :as json]
[app.util.time :as dt]
[buddy.sign.jwk :as jwk]
[buddy.sign.jwt :as jwt]
[clojure.set :as set]
@@ -514,7 +514,7 @@
[cfg info request]
(let [info (assoc info
:iss :prepared-register
:exp (ct/in-future {:hours 48}))
:exp (dt/in-future {:hours 48}))
params {:token (tokens/generate (::setup/props cfg) info)
:provider (:provider (:path-params request))
@@ -571,7 +571,7 @@
token (or (:invitation-token info)
(tokens/generate (::setup/props cfg)
{:iss :auth
:exp (ct/in-future "15m")
:exp (dt/in-future "15m")
:profile-id (:id profile)}))
props (audit/profile->props profile)
context (d/without-nils {:external-session-id (:external-session-id info)})]
@@ -619,7 +619,7 @@
:invitation-token (:invitation-token params)
:external-session-id esid
:props props
:exp (ct/in-future "4h")}
:exp (dt/in-future "4h")}
state (tokens/generate (::setup/props cfg)
(d/without-nils params))
uri (build-auth-uri cfg state)]

View File

@@ -15,33 +15,29 @@
[app.common.files.migrations :as fmg]
[app.common.files.validate :as fval]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
[app.common.weak :as weak]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as sql]
[app.features.fdata :as fdata]
[app.features.file-migrations :as fmigr]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :as feat.fmigr]
[app.loggers.audit :as-alias audit]
[app.loggers.webhooks :as-alias webhooks]
[app.storage :as sto]
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.set :as set]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io]
[promesa.exec :as px]))
[datoteka.io :as io]))
(set! *warn-on-reflection* true)
(def ^:dynamic *state* nil)
(def ^:dynamic *options* nil)
(def ^:dynamic *reference-file* nil)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; DEFAULTS
@@ -57,12 +53,17 @@
(* 1024 1024 100))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare get-resolved-file-libraries)
(declare update-file!)
(def file-attrs
(sm/keys ctf/schema:file))
#{:id
:name
:migrations
:features
:project-id
:is-shared
:version
:data})
(defn parse-file-format
[template]
@@ -150,33 +151,22 @@
changes (assoc :changes (blob/decode changes))
data (assoc :data (blob/decode data)))))
(def sql:get-minimal-file
"SELECT f.id,
f.revn,
f.modified_at,
f.deleted_at
FROM file AS f
WHERE f.id = ?")
(defn get-minimal-file
[cfg id & {:as opts}]
(db/get-with-sql cfg [sql:get-minimal-file id] opts))
(defn decode-file
"A general purpose file decoding function that resolves all external
pointers, run migrations and return plain vanilla file map"
[cfg {:keys [id] :as file} & {:keys [migrate?] :or {migrate? true}}]
(binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)]
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(let [file (->> file
(fmigr/resolve-applied-migrations cfg)
(fdata/resolve-file-data cfg))
(feat.fmigr/resolve-applied-migrations cfg)
(feat.fdata/resolve-file-data cfg))
libs (delay (get-resolved-file-libraries cfg file))]
(-> file
(update :features db/decode-pgarray #{})
(update :data blob/decode)
(update :data fdata/process-pointers deref)
(update :data fdata/process-objects (partial into {}))
(update :data feat.fdata/process-pointers deref)
(update :data feat.fdata/process-objects (partial into {}))
(update :data assoc :id id)
(cond-> migrate? (fmg/migrate-file libs))))))
@@ -188,9 +178,9 @@
and decoding."
[cfg file-id & {:as opts}]
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
(when-let [row (db/get* conn :file {:id file-id}
(assoc opts ::db/remove-deleted false))]
(decode-file cfg row opts)))))
(some->> (db/get* conn :file {:id file-id}
(assoc opts ::db/remove-deleted false))
(decode-file cfg)))))
(defn clean-file-features
[file]
@@ -431,27 +421,6 @@
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"])
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
(defn invalidate-thumbnails
[cfg file-id]
(let [storage (sto/resolve cfg)
sql-1
(str "update file_tagged_object_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")
sql-2
(str "update file_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")]
(run! #(sto/touch-object! storage %)
(sequence
(keep :media-id)
(concat
(db/exec! cfg [sql-1 file-id])
(db/exec! cfg [sql-2 file-id]))))))
(defn process-file
[cfg {:keys [id] :as file}]
(let [libs (delay (get-resolved-file-libraries cfg file))]
@@ -476,78 +445,77 @@
(vary-meta dissoc ::fmg/migrated))))
(defn encode-file
[{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}]
(let [file (if (and (contains? features "fdata/objects-map")
(:data file))
(fdata/enable-objects-map file)
[{:keys [::db/conn] :as cfg} {:keys [id features] :as file}]
(let [file (if (contains? features "fdata/objects-map")
(feat.fdata/enable-objects-map file)
file)
file (if (and (contains? features "fdata/pointer-map")
(:data file))
(binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
(let [file (fdata/enable-pointer-map file)]
(fdata/persist-pointers! cfg id)
file (if (contains? features "fdata/pointer-map")
(binding [pmap/*tracked* (pmap/create-tracked)]
(let [file (feat.fdata/enable-pointer-map file)]
(feat.fdata/persist-pointers! cfg id)
file))
file)]
(-> file
(d/update-when :features into-array)
(d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data)))))))
(update :features db/encode-pgarray conn "text")
(update :data blob/encode))))
(defn- file->params
(defn get-params-from-file
[file]
(-> (select-keys file file-attrs)
(dissoc :team-id)
(dissoc :migrations)))
(let [params {:has-media-trimmed (:has-media-trimmed file)
:ignore-sync-until (:ignore-sync-until file)
:project-id (:project-id file)
:features (:features file)
:name (:name file)
:is-shared (:is-shared file)
:version (:version file)
:data (:data file)
:id (:id file)
:deleted-at (:deleted-at file)
:created-at (:created-at file)
:modified-at (:modified-at file)
:revn (:revn file)
:vern (:vern file)}]
(-> (d/without-nils params)
(assoc :data-backend nil)
(assoc :data-ref-id nil))))
(defn insert-file!
"Insert a new file into the database table. Expectes a not-encoded file.
Returns nil."
"Insert a new file into the database table"
[{:keys [::db/conn] :as cfg} file & {:as opts}]
(when (:migrations file)
(fmigr/upsert-migrations! conn file))
(let [file (encode-file cfg file)]
(db/insert! conn :file
(file->params file)
{::db/return-keys false})
nil))
(feat.fmigr/upsert-migrations! conn file)
(let [params (-> (encode-file cfg file)
(get-params-from-file))]
(db/insert! conn :file params opts)))
(defn update-file!
"Update an existing file on the database. Expects not encoded file."
[{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}]
"Update an existing file on the database."
[{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}]
(let [file (encode-file cfg file)
params (-> (get-params-from-file file)
(dissoc :id))]
(if (::reset-migrations opts false)
(fmigr/reset-migrations! conn file)
(fmigr/upsert-migrations! conn file))
;; If file was already offloaded, we touch the underlying storage
;; object for properly trigger storage-gc-touched task
(when (feat.fdata/offloaded? file)
(some->> (:data-ref-id file) (sto/touch-object! storage)))
(let [file
(encode-file cfg file)
params
(file->params (dissoc file :id))]
(db/update! conn :file params
{:id id}
{::db/return-keys false})
nil))
(feat.fmigr/upsert-migrations! conn file)
(db/update! conn :file params {:id id} opts)))
(defn save-file!
"Applies all the final validations and perist the file, binfile
specific, should not be used outside of binfile domain.
Returns nil"
specific, should not be used outside of binfile domain"
[{:keys [::timestamp] :as cfg} file & {:as opts}]
(assert (ct/inst? timestamp) "expected valid timestamp")
(assert (dt/instant? timestamp) "expected valid timestamp")
(let [file (-> file
(assoc :created-at timestamp)
(assoc :modified-at timestamp)
(cond-> (not (::overwrite cfg))
(assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))))
(assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))
(update :features
(fn [features]
(-> (::features cfg #{})
@@ -564,9 +532,8 @@
(when (ex/exception? result)
(l/error :hint "file schema validation error" :cause result))))
(if (::overwrite cfg)
(update-file! cfg file (assoc opts ::reset-migrations true))
(insert-file! cfg file opts))))
(insert-file! cfg file opts)))
(def ^:private sql:get-file-libraries
"WITH RECURSIVE libs AS (
@@ -591,8 +558,7 @@
l.revn,
l.vern,
l.synced_at,
l.is_shared,
l.version
l.is_shared
FROM libs AS l
INNER JOIN project AS p ON (p.id = l.project_id)
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
@@ -608,18 +574,8 @@
(db/exec! conn [sql:get-file-libraries file-id])))
(defn get-resolved-file-libraries
"Get all file libraries including itself. Returns an instance of
LoadableWeakValueMap that allows do not have strong references to
the loaded libraries and reduce possible memory pressure on having
all this libraries loaded at same time on processing file validation
or file migration.
This still requires at least one library at time to be loaded while
access to it is performed, but it improves considerable not having
the need of loading all the libraries at the same time."
[{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
(let [library-ids (->> (get-file-libraries conn (:id file))
(map :id)
(cons (:id file)))
load-fn #(get-file cfg % :migrate? false)]
(weak/loadable-weak-value-map library-ids load-fn {id file})))
"A helper for preload file libraries"
[{:keys [::db/conn] :as cfg} file]
(->> (get-file-libraries conn (:id file))
(into [file] (map #(get-file cfg (:id %))))
(d/index-by :id)))

View File

@@ -36,6 +36,11 @@
"fdata/shape-data-type"
nil
;; There is no migration needed, but we don't want to allow
;; copy paste nor import of variant files into no-variant teams
"variants/v1"
nil
(ex/raise :type :internal
:code :no-migration-defined
:hint (str/ffmt "no migation for feature '%' on file importation" feature)

View File

@@ -17,7 +17,6 @@
[app.common.fressian :as fres]
[app.common.logging :as l]
[app.common.spec :as us]
[app.common.time :as ct]
[app.common.types.file :as ctf]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -31,6 +30,7 @@
[app.storage.tmp :as tmp]
[app.tasks.file-gc]
[app.util.events :as events]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.java.io :as jio]
[clojure.set :as set]
@@ -434,7 +434,7 @@
(defn read-import!
"Do the importation of the specified resource in penpot custom binary
format."
[{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (ct/now)} :as options}]
[{:keys [::bfc/input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}]
(dm/assert!
"expected input stream"
@@ -442,7 +442,7 @@
(dm/assert!
"expected valid instant"
(ct/inst? timestamp))
(dt/instant? timestamp))
(let [version (read-header! input)]
(read-import (assoc options ::version version ::bfc/timestamp timestamp))))
@@ -682,7 +682,7 @@
(io/coercible? output))
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
ab (volatile! false)
cs (volatile! nil)]
(try
@@ -720,7 +720,7 @@
(satisfies? jio/IOFactory input))
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
cs (volatile! nil)]
(l/info :hint "import: started" :id (str id))
@@ -742,6 +742,6 @@
(finally
(l/info :hint "import: terminated"
:id (str id)
:elapsed (ct/format-duration (tp))
:elapsed (dt/format-duration (tp))
:error? (some? @cs))))))

View File

@@ -13,7 +13,6 @@
[app.common.data :as d]
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -24,6 +23,7 @@
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.events :as events]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.set :as set]
[cuerdas.core :as str]
@@ -344,7 +344,7 @@
(defn export-team!
[cfg team-id]
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
cfg (create-database cfg)]
(l/inf :hint "start"
@@ -378,15 +378,15 @@
(l/inf :hint "end"
:operation "export"
:id (str id)
:elapsed (ct/format-duration elapsed)))))))
:elapsed (dt/format-duration elapsed)))))))
(defn import-team!
[cfg path]
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
cfg (-> (create-database cfg path)
(assoc ::bfc/timestamp (ct/now)))]
(assoc ::bfc/timestamp (dt/now)))]
(l/inf :hint "start"
:operation "import"
@@ -434,4 +434,4 @@
(l/inf :hint "end"
:operation "import"
:id (str id)
:elapsed (ct/format-duration elapsed)))))))
:elapsed (dt/format-duration elapsed)))))))

View File

@@ -20,14 +20,13 @@
[app.common.media :as cmedia]
[app.common.schema :as sm]
[app.common.thumbnails :as cth]
[app.common.time :as ct]
[app.common.types.color :as ctcl]
[app.common.types.component :as ctc]
[app.common.types.file :as ctf]
[app.common.types.page :as ctp]
[app.common.types.plugins :as ctpg]
[app.common.types.shape :as cts]
[app.common.types.tokens-lib :as ctob]
[app.common.types.tokens-lib :as cto]
[app.common.types.typography :as cty]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -36,15 +35,14 @@
[app.storage :as sto]
[app.storage.impl :as sto.impl]
[app.util.events :as events]
[app.util.time :as dt]
[clojure.java.io :as jio]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io])
(:import
java.io.File
java.io.InputStream
java.io.OutputStreamWriter
java.lang.AutoCloseable
java.util.zip.ZipEntry
java.util.zip.ZipFile
java.util.zip.ZipOutputStream))
@@ -94,7 +92,7 @@
(defn- default-now
[o]
(or o (ct/now)))
(or o (dt/now)))
;; --- ENCODERS
@@ -105,25 +103,25 @@
(sm/encoder ctp/schema:page sm/json-transformer))
(def encode-shape
(sm/encoder cts/schema:shape sm/json-transformer))
(sm/encoder ::cts/shape sm/json-transformer))
(def encode-media
(sm/encoder ctf/schema:media sm/json-transformer))
(sm/encoder ::ctf/media sm/json-transformer))
(def encode-component
(sm/encoder ctc/schema:component sm/json-transformer))
(sm/encoder ::ctc/component sm/json-transformer))
(def encode-color
(sm/encoder ctcl/schema:library-color sm/json-transformer))
(def encode-typography
(sm/encoder cty/schema:typography sm/json-transformer))
(sm/encoder ::cty/typography sm/json-transformer))
(def encode-tokens-lib
(sm/encoder ctob/schema:tokens-lib sm/json-transformer))
(sm/encoder ::cto/tokens-lib sm/json-transformer))
(def encode-plugin-data
(sm/encoder ctpg/schema:plugin-data sm/json-transformer))
(sm/encoder ::ctpg/plugin-data sm/json-transformer))
(def encode-storage-object
(sm/encoder schema:storage-object sm/json-transformer))
@@ -140,7 +138,7 @@
(sm/decoder ctf/schema:media sm/json-transformer))
(def decode-component
(sm/decoder ctc/schema:component sm/json-transformer))
(sm/decoder ::ctc/component sm/json-transformer))
(def decode-color
(sm/decoder ctcl/schema:library-color sm/json-transformer))
@@ -149,19 +147,19 @@
(sm/decoder schema:file sm/json-transformer))
(def decode-page
(sm/decoder ctp/schema:page sm/json-transformer))
(sm/decoder ::ctp/page sm/json-transformer))
(def decode-shape
(sm/decoder cts/schema:shape sm/json-transformer))
(sm/decoder ::cts/shape sm/json-transformer))
(def decode-typography
(sm/decoder cty/schema:typography sm/json-transformer))
(sm/decoder ::cty/typography sm/json-transformer))
(def decode-tokens-lib
(sm/decoder ctob/schema:tokens-lib sm/json-transformer))
(sm/decoder cto/schema:tokens-lib sm/json-transformer))
(def decode-plugin-data
(sm/decoder ctpg/schema:plugin-data sm/json-transformer))
(sm/decoder ::ctpg/plugin-data sm/json-transformer))
(def decode-storage-object
(sm/decoder schema:storage-object sm/json-transformer))
@@ -175,31 +173,31 @@
(sm/check-fn schema:manifest))
(def validate-file
(sm/check-fn ctf/schema:file))
(sm/check-fn ::ctf/file))
(def validate-page
(sm/check-fn ctp/schema:page))
(sm/check-fn ::ctp/page))
(def validate-shape
(sm/check-fn cts/schema:shape))
(sm/check-fn ::cts/shape))
(def validate-media
(sm/check-fn ctf/schema:media))
(sm/check-fn ::ctf/media))
(def validate-color
(sm/check-fn ctcl/schema:library-color))
(def validate-component
(sm/check-fn ctc/schema:component))
(sm/check-fn ::ctc/component))
(def validate-typography
(sm/check-fn cty/schema:typography))
(sm/check-fn ::cty/typography))
(def validate-tokens-lib
(sm/check-fn ctob/schema:tokens-lib))
(sm/check-fn ::cto/tokens-lib))
(def validate-plugin-data
(sm/check-fn ctpg/schema:plugin-data))
(sm/check-fn ::ctpg/plugin-data))
(def validate-storage-object
(sm/check-fn schema:storage-object))
@@ -253,9 +251,9 @@
(write-entry! output path params)
(with-open [input (sto/get-object-data storage sobject)]
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
(.putNextEntry output (ZipEntry. (str "objects/" id ext)))
(io/copy input output :size (:size sobject))
(.closeEntry ^ZipOutputStream output))))))
(.closeEntry output))))))
(defn- export-file
[{:keys [::file-id ::output] :as cfg}]
@@ -286,12 +284,10 @@
(assoc :options (:options data))
:always
(dissoc :data))
(dissoc :data)
file (cond-> file
:always
(encode-file))
path (str "files/" file-id ".json")]
(write-entry! output path file))
@@ -349,8 +345,7 @@
typography (encode-typography object)]
(write-entry! output path typography)))
(when (and tokens-lib
(not (ctob/empty-lib? tokens-lib)))
(when tokens-lib
(let [path (str "files/" file-id "/tokens.json")
encoded-tokens (encode-tokens-lib tokens-lib)]
(write-entry! output path encoded-tokens)))))
@@ -450,7 +445,7 @@
(defn- read-manifest
[^ZipFile input]
(let [entry (get-zip-entry input "manifest.json")]
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
(with-open [reader (zip-entry-reader input entry)]
(let [manifest (json/read reader :key-fn json/read-kebab-key)]
(decode-manifest manifest)))))
@@ -540,27 +535,24 @@
(defn- read-entry
[^ZipFile input entry]
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
(with-open [reader (zip-entry-reader input entry)]
(json/read reader :key-fn json/read-kebab-key)))
(defn- read-plain-entry
[^ZipFile input entry]
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
(with-open [reader (zip-entry-reader input entry)]
(json/read reader)))
(defn- read-file
[{:keys [::bfc/input ::bfc/timestamp]} file-id]
[{:keys [::bfc/input ::file-id]}]
(let [path (str "files/" file-id ".json")
entry (get-zip-entry input path)]
(-> (read-entry input entry)
(decode-file)
(update :revn d/nilv 1)
(update :created-at d/nilv timestamp)
(update :modified-at d/nilv timestamp)
(validate-file))))
(defn- read-file-plugin-data
[{:keys [::bfc/input]} file-id]
[{:keys [::bfc/input ::file-id]}]
(let [path (str "files/" file-id "/plugin-data.json")
entry (get-zip-entry* input path)]
(some->> entry
@@ -569,7 +561,7 @@
(validate-plugin-data))))
(defn- read-file-media
[{:keys [::bfc/input ::entries]} file-id]
[{:keys [::bfc/input ::file-id ::entries]}]
(->> (keep (match-media-entry-fn file-id) entries)
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry input entry)
@@ -589,7 +581,7 @@
(not-empty)))
(defn- read-file-colors
[{:keys [::bfc/input ::entries]} file-id]
[{:keys [::bfc/input ::file-id ::entries]}]
(->> (keep (match-color-entry-fn file-id) entries)
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry input entry)
@@ -602,7 +594,7 @@
(not-empty)))
(defn- read-file-components
[{:keys [::bfc/input ::entries]} file-id]
[{:keys [::bfc/input ::file-id ::entries]}]
(let [clean-component-post-decode
(fn [component]
(d/update-when component :objects
@@ -633,7 +625,7 @@
(not-empty))))
(defn- read-file-typographies
[{:keys [::bfc/input ::entries]} file-id]
[{:keys [::bfc/input ::file-id ::entries]}]
(->> (keep (match-typography-entry-fn file-id) entries)
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry input entry)
@@ -646,14 +638,14 @@
(not-empty)))
(defn- read-file-tokens-lib
[{:keys [::bfc/input ::entries]} file-id]
[{:keys [::bfc/input ::file-id ::entries]}]
(when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
(->> (read-plain-entry input entry)
(decode-tokens-lib)
(validate-tokens-lib))))
(defn- read-file-shapes
[{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
[{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}]
(->> (keep (match-shape-entry-fn file-id page-id) entries)
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry input entry)
@@ -667,14 +659,15 @@
(not-empty)))
(defn- read-file-pages
[{:keys [::bfc/input ::entries] :as cfg} file-id]
[{:keys [::bfc/input ::file-id ::entries] :as cfg}]
(->> (keep (match-page-entry-fn file-id) entries)
(keep (fn [{:keys [id entry]}]
(let [page (->> (read-entry input entry)
(decode-page))
page (dissoc page :options)]
(when (= id (:id page))
(let [objects (read-file-shapes cfg file-id id)]
(let [objects (-> (assoc cfg ::page-id id)
(read-file-shapes))]
(assoc page :objects objects))))))
(sort-by :index)
(reduce (fn [result {:keys [id] :as page}]
@@ -682,7 +675,7 @@
(d/ordered-map))))
(defn- read-file-thumbnails
[{:keys [::bfc/input ::entries] :as cfg} file-id]
[{:keys [::bfc/input ::file-id ::entries] :as cfg}]
(->> (keep (match-thumbnail-entry-fn file-id) entries)
(reduce (fn [result {:keys [page-id frame-id tag entry]}]
(let [object (->> (read-entry input entry)
@@ -697,13 +690,13 @@
(not-empty)))
(defn- read-file-data
[cfg file-id]
(let [colors (read-file-colors cfg file-id)
typographies (read-file-typographies cfg file-id)
tokens-lib (read-file-tokens-lib cfg file-id)
components (read-file-components cfg file-id)
plugin-data (read-file-plugin-data cfg file-id)
pages (read-file-pages cfg file-id)]
[cfg]
(let [colors (read-file-colors cfg)
typographies (read-file-typographies cfg)
tokens-lib (read-file-tokens-lib cfg)
components (read-file-components cfg)
plugin-data (read-file-plugin-data cfg)
pages (read-file-pages cfg)]
{:pages (-> pages keys vec)
:pages-index (into {} pages)
:colors colors
@@ -713,11 +706,11 @@
:plugin-data plugin-data}))
(defn- import-file
[{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}]
[{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}]
(let [file-id' (bfc/lookup-index file-id)
file (read-file cfg file-id)
media (read-file-media cfg file-id)
thumbnails (read-file-thumbnails cfg file-id)]
file (read-file cfg)
media (read-file-media cfg)
thumbnails (read-file-thumbnails cfg)]
(l/dbg :hint "processing file"
:id (str file-id')
@@ -747,7 +740,7 @@
(vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails))
(vswap! bfc/*state* update :thumbnails into thumbnails))
(let [data (-> (read-file-data cfg file-id)
(let [data (-> (read-file-data cfg)
(d/without-nils)
(assoc :id file-id')
(cond-> (:options file)
@@ -764,7 +757,7 @@
file (ctf/check-file file)]
(bfm/register-pending-migrations! cfg file)
(bfc/save-file! cfg file)
(bfc/save-file! cfg file ::db/return-keys false)
file-id')))
@@ -860,8 +853,7 @@
:file-id (str (:file-id params))
::l/sync? true)
(db/insert! conn :file-media-object params
::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))))
(db/insert! conn :file-media-object params))))
(defn- import-file-thumbnails
[{:keys [::db/conn] :as cfg}]
@@ -881,77 +873,17 @@
:media-id (str media-id)
::l/sync? true)
(db/insert! conn :file-tagged-object-thumbnail params
{::db/on-conflict-do-nothing? true}))))
(defn- import-files*
[{:keys [::manifest] :as cfg}]
(bfc/disable-database-timeouts! cfg)
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
(let [files (get manifest :files)
result (reduce (fn [result {:keys [id] :as file}]
(let [name' (get file :name)
name' (if (map? name)
(get name id)
name')
file (assoc file :name name')]
(conj result (import-file cfg file))))
[]
files)]
(import-file-relations cfg)
(import-storage-objects cfg)
(import-file-media cfg)
(import-file-thumbnails cfg)
(bfm/apply-pending-migrations! cfg)
result))
(defn- import-file-and-overwrite*
[{:keys [::manifest ::bfc/file-id] :as cfg}]
(when (not= 1 (count (:files manifest)))
(ex/raise :type :validation
:code :invalid-condition
:hint "unable to perform in-place update with binfile containing more than 1 file"
:manifest manifest))
(bfc/disable-database-timeouts! cfg)
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
file (first (get manifest :files))
cfg (assoc cfg ::bfc/overwrite true)]
(vswap! bfc/*state* update :index assoc (:id file) file-id)
(binding [bfc/*options* cfg
bfc/*reference-file* ref-file]
(import-file cfg file)
(import-storage-objects cfg)
(import-file-media cfg)
(bfc/invalidate-thumbnails cfg file-id)
(bfm/apply-pending-migrations! cfg)
[file-id])))
(db/insert! conn :file-tagged-object-thumbnail params))))
(defn- import-files
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
[{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}]
(assert (instance? ZipFile input) "expected zip file")
(assert (ct/inst? timestamp) "expected valid instant")
(assert (dt/instant? timestamp) "expected valid instant")
(let [manifest (-> (read-manifest input)
(validate-manifest))
entries (read-zip-entries input)
cfg (-> cfg
(assoc ::entries entries)
(assoc ::manifest manifest)
(assoc ::bfc/timestamp timestamp))]
entries (read-zip-entries input)]
(when-not (= "penpot/export-files" (:type manifest))
(ex/raise :type :validation
@@ -959,6 +891,7 @@
:hint "unexpected type on manifest"
:manifest manifest))
;; Check if all files referenced on manifest are present
(doseq [{file-id :id features :features} (:files manifest)]
(let [path (str "files/" file-id ".json")]
@@ -974,10 +907,35 @@
(events/tap :progress {:section :manifest})
(binding [bfc/*state* (volatile! {:media [] :index {}})]
(if (::bfc/file-id cfg)
(db/tx-run! cfg import-file-and-overwrite*)
(db/tx-run! cfg import-files*)))))
(let [index (bfc/update-index (map :id (:files manifest)))
state {:media [] :index index}
cfg (-> cfg
(assoc ::entries entries)
(assoc ::manifest manifest)
(assoc ::bfc/timestamp timestamp))]
(binding [bfc/*state* (volatile! state)]
(db/tx-run! cfg (fn [cfg]
(bfc/disable-database-timeouts! cfg)
(let [ids (->> (:files manifest)
(reduce (fn [result {:keys [id] :as file}]
(let [name' (get file :name)
name' (if (map? name)
(get name id)
name')]
(conj result (-> cfg
(assoc ::file-id id)
(assoc ::file-name name')
(import-file)))))
[]))]
(import-file-relations cfg)
(import-storage-objects cfg)
(import-file-media cfg)
(import-file-thumbnails cfg)
(bfm/apply-pending-migrations! cfg)
ids)))))))
;; --- PUBLIC API
@@ -1003,14 +961,14 @@
"expected instance of jio/IOFactory for `input`")
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
ab (volatile! false)
cs (volatile! nil)]
(try
(l/info :hint "start exportation" :export-id (str id))
(binding [bfc/*state* (volatile! (bfc/initial-state))]
(with-open [^AutoCloseable output (io/output-stream output)]
(with-open [^AutoCloseable output (ZipOutputStream. output)]
(with-open [output (io/output-stream output)]
(with-open [output (ZipOutputStream. output)]
(let [cfg (assoc cfg ::output output)]
(export-files cfg)
(export-storage-objects cfg)))))
@@ -1049,12 +1007,12 @@
"expected instance of jio/IOFactory for `input`")
(let [id (uuid/next)
tp (ct/tpoint)
tp (dt/tpoint)
cs (volatile! nil)]
(l/info :hint "import: started" :id (str id))
(try
(with-open [input (ZipFile. ^File (fs/file input))]
(with-open [input (ZipFile. (fs/file input))]
(import-files (assoc cfg ::bfc/input input)))
(catch Throwable cause
@@ -1064,11 +1022,11 @@
(finally
(l/info :hint "import: terminated"
:id (str id)
:elapsed (ct/format-duration (tp))
:elapsed (dt/format-duration (tp))
:error? (some? @cs))))))
(defn get-manifest
[path]
(with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
(with-open [input (ZipFile. (fs/file path))]
(-> (read-manifest input)
(validate-manifest))))

View File

@@ -12,10 +12,10 @@
[app.common.exceptions :as ex]
[app.common.flags :as flags]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.version :as v]
[app.util.overrides]
[app.util.time :as dt]
[clojure.core :as c]
[clojure.java.io :as io]
[cuerdas.core :as str]
@@ -59,10 +59,10 @@
:smtp-default-reply-to "Penpot <no-reply@example.com>"
:smtp-default-from "Penpot <no-reply@example.com>"
:profile-complaint-max-age (ct/duration {:days 7})
:profile-complaint-max-age (dt/duration {:days 7})
:profile-complaint-threshold 2
:profile-bounce-max-age (ct/duration {:days 7})
:profile-bounce-max-age (dt/duration {:days 7})
:profile-bounce-threshold 10
:telemetry-uri "https://telemetry.penpot.app/"
@@ -102,10 +102,10 @@
[:telemetry-with-taiga {:optional true} ::sm/boolean] ;; DELETE
[:auto-file-snapshot-every {:optional true} ::sm/int]
[:auto-file-snapshot-timeout {:optional true} ::ct/duration]
[:auto-file-snapshot-timeout {:optional true} ::dt/duration]
[:media-max-file-size {:optional true} ::sm/int]
[:deletion-delay {:optional true} ::ct/duration] ;; REVIEW
[:deletion-delay {:optional true} ::dt/duration] ;; REVIEW
[:telemetry-enabled {:optional true} ::sm/boolean]
[:default-blob-version {:optional true} ::sm/int]
[:allow-demo-users {:optional true} ::sm/boolean]
@@ -148,10 +148,10 @@
[:auth-data-cookie-domain {:optional true} :string]
[:auth-token-cookie-name {:optional true} :string]
[:auth-token-cookie-max-age {:optional true} ::ct/duration]
[:auth-token-cookie-max-age {:optional true} ::dt/duration]
[:registration-domain-whitelist {:optional true} [::sm/set :string]]
[:email-verify-threshold {:optional true} ::ct/duration]
[:email-verify-threshold {:optional true} ::dt/duration]
[:github-client-id {:optional true} :string]
[:github-client-secret {:optional true} :string]
@@ -186,9 +186,9 @@
[:ldap-starttls {:optional true} ::sm/boolean]
[:ldap-user-query {:optional true} :string]
[:profile-bounce-max-age {:optional true} ::ct/duration]
[:profile-bounce-max-age {:optional true} ::dt/duration]
[:profile-bounce-threshold {:optional true} ::sm/int]
[:profile-complaint-max-age {:optional true} ::ct/duration]
[:profile-complaint-max-age {:optional true} ::dt/duration]
[:profile-complaint-threshold {:optional true} ::sm/int]
[:redis-uri {:optional true} ::sm/uri]
@@ -298,7 +298,7 @@
(defn get-deletion-delay
[]
(or (c/get config :deletion-delay)
(ct/duration {:days 7})))
(dt/duration {:days 7})))
(defn get
"A configuration getter. Helps code be more testable."

View File

@@ -10,20 +10,19 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.geom.point :as gpt]
[app.common.json :as json]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.db.sql :as sql]
[app.metrics :as mtx]
[app.util.json :as json]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.set :as set]
[integrant.core :as ig]
[next.jdbc :as jdbc]
[next.jdbc.date-time :as jdbc-dt]
[next.jdbc.prepare :as jdbc.prepare]
[next.jdbc.transaction])
(:import
com.zaxxer.hikari.HikariConfig
@@ -34,7 +33,6 @@
java.io.InputStream
java.io.OutputStream
java.sql.Connection
java.sql.PreparedStatement
java.sql.Savepoint
org.postgresql.PGConnection
org.postgresql.geometric.PGpoint
@@ -379,9 +377,9 @@
(defn is-row-deleted?
[{:keys [deleted-at]}]
(and (ct/inst? deleted-at)
(and (dt/instant? deleted-at)
(< (inst-ms deleted-at)
(inst-ms (ct/now)))))
(inst-ms (dt/now)))))
(defn get*
"Retrieve a single row from database that matches a simple filters. Do
@@ -406,24 +404,6 @@
:hint "database object not found"))
row))
(defn get-with-sql
[ds sql & {:as opts}]
(let [rows (cond->> (exec! ds sql opts)
(::remove-deleted opts true)
(remove is-row-deleted?)
:always
(not-empty))]
(when (and (not rows) (::throw-if-not-exists opts true))
(ex/raise :type :not-found
:code :object-not-found
:hint "database object not found"))
(first rows)))
(def ^:private default-plan-opts
(-> default-opts
(assoc :fetch-size 1000)
@@ -605,7 +585,7 @@
(string? o)
(pginterval o)
(ct/duration? o)
(dt/duration? o)
(interval (inst-ms o))
:else
@@ -619,7 +599,7 @@
val (.getValue o)]
(if (or (= typ "json")
(= typ "jsonb"))
(json/decode val :key-fn keyword)
(json/decode val)
val))))
(defn decode-transit-pgobject
@@ -660,7 +640,7 @@
(when data
(doto (org.postgresql.util.PGobject.)
(.setType "jsonb")
(.setValue (json/encode data)))))
(.setValue (json/encode-str data)))))
;; --- Locks
@@ -706,8 +686,3 @@
[cause]
(and (sql-exception? cause)
(= "40001" (.getSQLState ^java.sql.SQLException cause))))
(extend-protocol jdbc.prepare/SettableParameter
clojure.lang.Keyword
(set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i]
(.setObject s i ^String (d/name v))))

View File

@@ -53,15 +53,8 @@
opts (cond-> opts
(::order-by opts) (assoc :order-by (::order-by opts))
(::columns opts) (assoc :columns (::columns opts))
(or (::db/for-update opts)
(::for-update opts))
(assoc :suffix "FOR UPDATE")
(or (::db/for-share opts)
(::for-share opts))
(assoc :suffix "FOR SHARE"))]
(::for-update opts) (assoc :suffix "FOR UPDATE")
(::for-share opts) (assoc :suffix "FOR SHARE"))]
(sql/for-query table where-params opts))))
(defn update

View File

@@ -12,16 +12,13 @@
[app.common.files.helpers :as cfh]
[app.common.files.migrations :as fmg]
[app.common.logging :as l]
[app.common.types.objects-map :as omap]
[app.common.types.path :as path]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.storage :as sto]
[app.util.blob :as blob]
[app.util.objects-map :as omap.legacy]
[app.util.pointer-map :as pmap]
[app.worker :as wrk]
[promesa.exec :as px]))
[app.util.objects-map :as omap]
[app.util.pointer-map :as pmap]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; OFFLOAD
@@ -39,7 +36,10 @@
[file & _opts]
(let [update-page
(fn [page]
(update page :objects omap/wrap))
(if (and (pmap/pointer-map? page)
(not (pmap/loaded? page)))
page
(update page :objects omap/wrap)))
update-data
(fn [fdata]
@@ -49,20 +49,6 @@
(update :data update-data)
(update :features conj "fdata/objects-map"))))
(defn disable-objects-map
[file & _opts]
(let [update-page
(fn [page]
(update page :objects #(into {} %)))
update-data
(fn [fdata]
(update fdata :pages-index d/update-vals update-page))]
(-> file
(update :data update-data)
(update :features disj "fdata/objects-map"))))
(defn process-objects
"Apply a function to all objects-map on the file. Usualy used for convert
the objects-map instances to plain maps"
@@ -72,8 +58,7 @@
(fn [page]
(update page :objects
(fn [objects]
(if (or (omap/objects-map? objects)
(omap.legacy/objects-map? objects))
(if (omap/objects-map? objects)
(update-fn objects)
objects)))))
fdata))
@@ -96,12 +81,6 @@
(let [data (get-file-data system file)]
(assoc file :data data)))
(defn decode-file-data
[{:keys [::wrk/executor]} {:keys [data] :as file}]
(cond-> file
(bytes? data)
(assoc :data (px/invoke! executor #(blob/decode data)))))
(defn load-pointer
"A database loader pointer helper"
[system file-id id]

View File

@@ -8,7 +8,6 @@
"Backend specific code for file migrations. Implemented as permanent feature of files."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.files.migrations :as fmg :refer [xf:map-name]]
[app.db :as db]
[app.db.sql :as-alias sql]))
@@ -27,19 +26,12 @@
(defn upsert-migrations!
"Persist or update file migrations. Return the updated/inserted number
of rows"
[cfg {:keys [id] :as file}]
(let [conn (db/get-connection cfg)
migrations (or (-> file meta ::fmg/migrated)
(-> file :migrations))
[conn {:keys [id] :as file}]
(let [migrations (or (-> file meta ::fmg/migrated)
(-> file :migrations not-empty)
fmg/available-migrations)
columns [:file-id :name]
rows (->> migrations
(mapv (fn [name] [id name]))
(not-empty))]
(when-not rows
(ex/raise :type :internal
:code :missing-migrations
:hint "no migrations available on file"))
rows (mapv (fn [name] [id name]) migrations)]
(-> (db/insert-many! conn :file-migration columns rows
{::db/return-keys false
@@ -48,6 +40,6 @@
(defn reset-migrations!
"Replace file migrations"
[cfg {:keys [id] :as file}]
(db/delete! cfg :file-migration {:file-id id})
(upsert-migrations! cfg file))
[conn {:keys [id] :as file}]
(db/delete! conn :file-migration {:file-id id})
(upsert-migrations! conn file))

View File

@@ -7,23 +7,22 @@
(ns app.features.logical-deletion
"A code related to handle logical deletion mechanism"
(:require
[app.common.time :as ct]
[app.config :as cf]))
(def ^:private canceled-status
#{"canceled" "unpaid"})
[app.config :as cf]
[app.util.time :as dt]))
(defn get-deletion-delay
"Calculate the next deleted-at for a resource (file, team, etc) in function
of team settings"
[team]
(if-let [{:keys [type status]} (get team :subscription)]
(if-let [subscription (get team :subscription)]
(cond
(and (= "unlimited" type) (not (contains? canceled-status status)))
(ct/duration {:days 30})
(and (= (:type subscription) "unlimited")
(= (:status subscription) "active"))
(dt/duration {:days 30})
(and (= "enterprise" type) (not (contains? canceled-status status)))
(ct/duration {:days 90})
(and (= (:type subscription) "enterprise")
(= (:status subscription) "active"))
(dt/duration {:days 90})
:else
(cf/get-deletion-delay))

View File

@@ -17,7 +17,6 @@
[app.http.awsns :as-alias awsns]
[app.http.debug :as-alias debug]
[app.http.errors :as errors]
[app.http.management :as mgmt]
[app.http.middleware :as mw]
[app.http.session :as session]
[app.http.websocket :as-alias ws]
@@ -26,7 +25,6 @@
[app.rpc :as-alias rpc]
[app.rpc.doc :as-alias rpc.doc]
[app.setup :as-alias setup]
[app.worker :as wrk]
[integrant.core :as ig]
[promesa.exec :as px]
[reitit.core :as r]
@@ -65,16 +63,15 @@
(assert (sm/check schema:server-params params)))
(defmethod ig/init-key ::server
[_ {:keys [::handler ::router ::host ::port ::wrk/executor] :as cfg}]
[_ {:keys [::handler ::router ::host ::port] :as cfg}]
(l/info :hint "starting http server" :port port :host host)
(let [options {:http/port port
:http/host host
:http/max-body-size (::max-body-size cfg)
:http/max-multipart-body-size (::max-multipart-body-size cfg)
:xnio/direct-buffers false
:xnio/io-threads (or (::io-threads cfg)
(max 3 (px/get-available-processors)))
:xnio/dispatch executor
:xnio/dispatch :virtual
:ring/compat :ring2
:socket/backlog 4069}
@@ -144,7 +141,6 @@
[::debug/routes schema:routes]
[::mtx/routes schema:routes]
[::awsns/routes schema:routes]
[::mgmt/routes schema:routes]
::session/manager
::setup/props
::db/pool])
@@ -172,9 +168,6 @@
["/webhooks"
(::awsns/routes cfg)]
["/management"
(::mgmt/routes cfg)]
(::ws/routes cfg)
["/api" {:middleware [[mw/cors]]}

View File

@@ -9,18 +9,18 @@
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.common.uri :as u]
[app.db :as db]
[app.storage :as sto]
[app.util.time :as dt]
[integrant.core :as ig]
[yetti.response :as-alias yres]))
(def ^:private cache-max-age
(ct/duration {:hours 24}))
(dt/duration {:hours 24}))
(def ^:private signature-max-age
(ct/duration {:hours 24 :minutes 15}))
(dt/duration {:hours 24 :minutes 15}))
(defn get-id
[{:keys [path-params]}]

View File

@@ -15,7 +15,6 @@
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.pprint :as pp]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -32,6 +31,7 @@
[app.storage.tmp :as tmp]
[app.util.blob :as blob]
[app.util.template :as tmpl]
[app.util.time :as dt]
[cuerdas.core :as str]
[datoteka.io :as io]
[emoji.core :as emj]
@@ -137,7 +137,7 @@
file (some-> params :file :path io/read* t/decode)]
(if (and file project-id)
(let [fname (str "Imported: " (:name file) "(" (ct/now) ")")
(let [fname (str "Imported: " (:name file) "(" (dt/now) ")")
reuse-id? (contains? params :reuseid)
file-id (or (and reuse-id? (ex/ignoring (-> params :file :filename parse-uuid)))
(uuid/next))]
@@ -222,7 +222,7 @@
(-> (io/resource "app/templates/error-report.v3.tmpl")
(tmpl/render (-> content
(assoc :id id)
(assoc :created-at (ct/format-inst created-at :rfc1123))))))]
(assoc :created-at (dt/format-instant created-at :rfc1123))))))]
(if-let [report (get-report request)]
(let [result (case (:version report)
@@ -246,7 +246,7 @@
(defn error-list-handler
[{:keys [::db/pool]} _request]
(let [items (->> (db/exec! pool [sql:error-reports])
(map #(update % :created-at ct/format-inst :rfc1123)))]
(map #(update % :created-at dt/format-instant :rfc1123)))]
{::yres/status 200
::yres/body (-> (io/resource "app/templates/error-list.tmpl")
(tmpl/render {:items items}))

View File

@@ -1,234 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.http.management
"Internal mangement HTTP API"
(:require
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.schema.generators :as sg]
[app.common.time :as ct]
[app.db :as db]
[app.main :as-alias main]
[app.rpc.commands.profile :as cmd.profile]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.worker :as-alias wrk]
[integrant.core :as ig]
[yetti.response :as-alias yres]))
;; ---- ROUTES
(declare ^:private authenticate)
(declare ^:private get-customer)
(declare ^:private update-customer)
(defmethod ig/assert-key ::routes
[_ params]
(assert (db/pool? (::db/pool params)) "expect valid database pool"))
(def ^:private default-system
{:name ::default-system
:compile
(fn [_ _]
(fn [handler cfg]
(fn [request]
(handler cfg request))))})
(def ^:private transaction
{:name ::transaction
:compile
(fn [data _]
(when (:transaction data)
(fn [handler]
(fn [cfg request]
(db/tx-run! cfg handler request)))))})
(defmethod ig/init-key ::routes
[_ cfg]
["" {:middleware [[default-system cfg]
[transaction]]}
["/authenticate"
{:handler authenticate
:allowed-methods #{:post}}]
["/get-customer"
{:handler get-customer
:transaction true
:allowed-methods #{:post}}]
["/update-customer"
{:handler update-customer
:allowed-methods #{:post}
:transaction true}]])
;; ---- HELPERS
(defn- coercer
[schema & {:as opts}]
(let [decode-fn (sm/decoder schema sm/json-transformer)
check-fn (sm/check-fn schema opts)]
(fn [data]
(-> data decode-fn check-fn))))
;; ---- API: AUTHENTICATE
(defn- authenticate
[cfg request]
(let [token (-> request :params :token)
props (get cfg ::setup/props)
result (tokens/verify props {:token token :iss "authentication"})]
{::yres/status 200
::yres/body result}))
;; ---- API: GET-CUSTOMER
(def ^:private schema:get-customer
[:map [:id ::sm/uuid]])
(def ^:private coerce-get-customer-params
(coercer schema:get-customer
:type :validation
:hint "invalid data provided for `get-customer` rpc call"))
(def ^:private sql:get-customer-slots
"WITH teams AS (
SELECT tpr.team_id AS id,
tpr.profile_id AS profile_id
FROM team_profile_rel AS tpr
WHERE tpr.is_owner IS true
AND tpr.profile_id = ?
), teams_with_slots AS (
SELECT tpr.team_id AS id,
count(*) AS total
FROM team_profile_rel AS tpr
WHERE tpr.team_id IN (SELECT id FROM teams)
AND tpr.can_edit IS true
GROUP BY 1
ORDER BY 2
)
SELECT max(total) AS total FROM teams_with_slots;")
(defn- get-customer-slots
[cfg profile-id]
(let [result (db/exec-one! cfg [sql:get-customer-slots profile-id])]
(:total result)))
(defn- get-customer
[cfg request]
(let [profile-id (-> request :params coerce-get-customer-params :id)
profile (cmd.profile/get-profile cfg profile-id)
result {:id (get profile :id)
:name (get profile :fullname)
:email (get profile :email)
:num-editors (get-customer-slots cfg profile-id)
:subscription (-> profile :props :subscription)}]
{::yres/status 200
::yres/body result}))
;; ---- API: UPDATE-CUSTOMER
(def ^:private schema:timestamp
(sm/type-schema
{:type ::timestamp
:pred ct/inst?
:type-properties
{:title "inst"
:description "The same as :app.common.time/inst but encodes to epoch"
:error/message "should be an instant"
:gen/gen (->> (sg/small-int)
(sg/fmap (fn [v] (ct/inst v))))
:decode/string ct/inst
:encode/string inst-ms
:decode/json ct/inst
:encode/json inst-ms}}))
(def ^:private schema:subscription
[:map {:title "Subscription"}
[:id ::sm/text]
[:customer-id ::sm/text]
[:type [:enum
"unlimited"
"professional"
"enterprise"]]
[:status [:enum
"active"
"canceled"
"incomplete"
"incomplete_expired"
"past_due"
"paused"
"trialing"
"unpaid"]]
[:billing-period [:enum
"month"
"day"
"week"
"year"]]
[:quantity :int]
[:description [:maybe ::sm/text]]
[:created-at schema:timestamp]
[:start-date [:maybe schema:timestamp]]
[:ended-at [:maybe schema:timestamp]]
[:trial-end [:maybe schema:timestamp]]
[:trial-start [:maybe schema:timestamp]]
[:cancel-at [:maybe schema:timestamp]]
[:canceled-at [:maybe schema:timestamp]]
[:current-period-end [:maybe schema:timestamp]]
[:current-period-start [:maybe schema:timestamp]]
[:cancel-at-period-end :boolean]
[:cancellation-details
[:map {:title "CancellationDetails"}
[:comment [:maybe ::sm/text]]
[:reason [:maybe ::sm/text]]
[:feedback [:maybe
[:enum
"customer_service"
"low_quality"
"missing_feature"
"other"
"switched_service"
"too_complex"
"too_expensive"
"unused"]]]]]])
(def ^:private schema:update-customer
[:map
[:id ::sm/uuid]
[:subscription [:maybe schema:subscription]]])
(def ^:private coerce-update-customer-params
(coercer schema:update-customer
:type :validation
:hint "invalid data provided for `update-customer` rpc call"))
(defn- update-customer
[cfg request]
(let [{:keys [id subscription]}
(-> request :params coerce-update-customer-params)
{:keys [props] :as profile}
(cmd.profile/get-profile cfg id ::db/for-update true)
props
(assoc props :subscription subscription)]
(l/dbg :hint "update customer"
:profile-id (str id)
:subscription-type (get subscription :type)
:subscription-status (get subscription :status)
:subscription-quantity (get subscription :quantity))
(db/update! cfg :profile
{:props (db/tjson props)}
{:id id}
{::db/return-keys false})
{::yres/status 201
::yres/body nil}))

View File

@@ -10,7 +10,6 @@
[app.common.data :as d]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.config :as cf]
[app.db :as db]
@@ -19,6 +18,7 @@
[app.main :as-alias main]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.util.time :as dt]
[cuerdas.core :as str]
[integrant.core :as ig]
[yetti.request :as yreq]))
@@ -35,10 +35,10 @@
(def default-auth-data-cookie-name "auth-data")
;; Default value for cookie max-age
(def default-cookie-max-age (ct/duration {:days 7}))
(def default-cookie-max-age (dt/duration {:days 7}))
;; Default age for automatic session renewal
(def default-renewal-max-age (ct/duration {:hours 6}))
(def default-renewal-max-age (dt/duration {:hours 6}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PROTOCOLS
@@ -66,7 +66,7 @@
[:map {:title "session-params"}
[:user-agent ::sm/text]
[:profile-id ::sm/uuid]
[:created-at ::ct/inst]])
[:created-at ::sm/inst]])
(def ^:private valid-params?
(sm/validator schema:params))
@@ -95,7 +95,7 @@
params))
(update! [_ params]
(let [updated-at (ct/now)]
(let [updated-at (dt/now)]
(db/update! pool :http-session
{:updated-at updated-at}
{:id (:id params)})
@@ -118,7 +118,7 @@
params))
(update! [_ params]
(let [updated-at (ct/now)]
(let [updated-at (dt/now)]
(swap! cache update (:id params) assoc :updated-at updated-at)
(assoc params :updated-at updated-at)))
@@ -158,7 +158,7 @@
(let [uagent (yreq/get-header request "user-agent")
params {:profile-id profile-id
:user-agent uagent
:created-at (ct/now)}
:created-at (dt/now)}
token (gen-token props params)
session (write! manager token params)]
(l/trace :hint "create" :profile-id (str profile-id))
@@ -203,8 +203,8 @@
(defn- renew-session?
[{:keys [updated-at] :as session}]
(and (ct/inst? updated-at)
(let [elapsed (ct/diff updated-at (ct/now))]
(and (dt/instant? updated-at)
(let [elapsed (dt/diff updated-at (dt/now))]
(neg? (compare default-renewal-max-age elapsed)))))
(defn- wrap-soft-auth
@@ -256,14 +256,14 @@
(defn- assign-auth-token-cookie
[response {token :id updated-at :updated-at}]
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
created-at (or updated-at (ct/now))
renewal (ct/plus created-at default-renewal-max-age)
expires (ct/plus created-at max-age)
created-at (or updated-at (dt/now))
renewal (dt/plus created-at default-renewal-max-age)
expires (dt/plus created-at max-age)
secure? (contains? cf/flags :secure-session-cookies)
strict? (contains? cf/flags :strict-session-cookies)
cors? (contains? cf/flags :cors)
name (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
comment (str "Renewal at: " (ct/format-inst renewal :rfc1123))
comment (str "Renewal at: " (dt/format-instant renewal :rfc1123))
cookie {:path "/"
:http-only true
:expires expires
@@ -279,11 +279,11 @@
domain (cf/get :auth-data-cookie-domain)
cname default-auth-data-cookie-name
created-at (or updated-at (ct/now))
renewal (ct/plus created-at default-renewal-max-age)
expires (ct/plus created-at max-age)
created-at (or updated-at (dt/now))
renewal (dt/plus created-at default-renewal-max-age)
expires (dt/plus created-at max-age)
comment (str "Renewal at: " (ct/format-inst renewal :rfc1123))
comment (str "Renewal at: " (dt/format-instant renewal :rfc1123))
secure? (contains? cf/flags :secure-session-cookies)
strict? (contains? cf/flags :strict-session-cookies)
cors? (contains? cf/flags :cors)
@@ -323,7 +323,7 @@
(defmethod ig/assert-key ::tasks/gc
[_ params]
(assert (db/pool? (::db/pool params)) "expected valid database pool")
(assert (ct/duration? (::tasks/max-age params))))
(assert (dt/duration? (::tasks/max-age params))))
(defmethod ig/expand-key ::tasks/gc
[k v]

View File

@@ -33,7 +33,7 @@
(println "event:" (d/name name))
(println "data:" (t/encode-str data {:type :json-verbose}))
(println))]
(.getBytes ^String data "UTF-8"))
(.getBytes data "UTF-8"))
(catch Throwable cause
(l/err :hint "unexpected error on encoding value on sse stream"
:cause cause)
@@ -44,8 +44,7 @@
(def default-headers
{"Content-Type" "text/event-stream;charset=UTF-8"
"Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
"Pragma" "no-cache"
"X-Accel-Buffering" "no"})
"Pragma" "no-cache"})
(defn response
[handler & {:keys [buf] :or {buf 32} :as opts}]

View File

@@ -11,12 +11,12 @@
[app.common.logging :as l]
[app.common.pprint :as pp]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.db :as db]
[app.http.session :as session]
[app.metrics :as mtx]
[app.msgbus :as mbus]
[app.util.time :as dt]
[app.util.websocket :as ws]
[integrant.core :as ig]
[promesa.exec.csp :as sp]
@@ -239,7 +239,7 @@
(defn- on-connect
[{:keys [::mtx/metrics]} {:keys [::ws/id] :as wsp}]
(let [created-at (ct/now)]
(let [created-at (dt/now)]
(l/trace :fn "on-connect" :conn-id id)
(swap! state assoc id wsp)
(mtx/run! metrics
@@ -253,7 +253,7 @@
(mtx/run! metrics :id :websocket-active-connections :dec 1)
(mtx/run! metrics
:id :websocket-session-timing
:val (/ (inst-ms (ct/diff created-at (ct/now))) 1000.0))))))
:val (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0))))))
(defn- on-rcv-message
[{:keys [::mtx/metrics ::profile-id ::session-id]} message]

View File

@@ -11,7 +11,6 @@
[app.common.data.macros :as dm]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -24,6 +23,7 @@
[app.setup :as-alias setup]
[app.util.inet :as inet]
[app.util.services :as-alias sv]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]))
@@ -108,9 +108,9 @@
[::ip-addr {:optional true} ::sm/text]
[::props {:optional true} [:map-of :keyword :any]]
[::context {:optional true} [:map-of :keyword :any]]
[::tracked-at {:optional true} ::ct/inst]
[::tracked-at {:optional true} ::sm/inst]
[::webhooks/event? {:optional true} ::sm/boolean]
[::webhooks/batch-timeout {:optional true} ::ct/duration]
[::webhooks/batch-timeout {:optional true} ::dt/duration]
[::webhooks/batch-key {:optional true}
[:or ::sm/fn ::sm/text :keyword]]])
@@ -199,7 +199,7 @@
(defn- handle-event!
[cfg event]
(let [params (event->params event)
tnow (ct/now)]
tnow (dt/now)]
(when (contains? cf/flags :audit-log)
;; NOTE: this operation may cause primary key conflicts on inserts
@@ -273,7 +273,7 @@
(let [event (-> (d/without-nils event)
(check-event))]
(db/run! cfg (fn [cfg]
(let [tnow (ct/now)
(let [tnow (dt/now)
params (-> (event->params event)
(assoc :created-at tnow)
(update :tracked-at #(or % tnow)))]

View File

@@ -9,7 +9,6 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -17,6 +16,7 @@
[app.http.client :as http]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.util.time :as dt]
[integrant.core :as ig]
[lambdaisland.uri :as u]
[promesa.exec :as px]))
@@ -55,7 +55,7 @@
[{:keys [::uri] :as cfg} events]
(let [token (tokens/generate (::setup/props cfg)
{:iss "authentication"
:iat (ct/now)
:iat (dt/now)
:uid uuid/zero})
body (t/encode {:events events})
headers {"content-type" "application/transit+json"

View File

@@ -10,13 +10,13 @@
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uri :as uri]
[app.config :as cf]
[app.db :as db]
[app.http.client :as http]
[app.loggers.audit :as audit]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.data.json :as json]
[cuerdas.core :as str]
@@ -124,7 +124,7 @@
{:id (:id whook)})))
(db/update! pool :webhook
{:updated-at (ct/now)
{:updated-at (dt/now)
:error-code nil
:error-count 0}
{:id (:id whook)})))
@@ -132,7 +132,7 @@
(report-delivery! [whook req rsp err]
(db/insert! pool :webhook-delivery
{:webhook-id (:id whook)
:created-at (ct/now)
:created-at (dt/now)
:error-code err
:req-data (db/tjson req)
:rsp-data (db/tjson rsp)}))]
@@ -155,7 +155,7 @@
(let [req {:uri (:uri whook)
:headers {"content-type" (:mtype whook)
"user-agent" (str/ffmt "penpot/%" (:main cf/version))}
:timeout (ct/duration "4s")
:timeout (dt/duration "4s")
:method :post
:body body}]
(try

View File

@@ -11,7 +11,6 @@
[app.auth.oidc.providers :as-alias oidc.providers]
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as-alias db]
[app.email :as-alias email]
@@ -20,7 +19,6 @@
[app.http.awsns :as http.awsns]
[app.http.client :as-alias http.client]
[app.http.debug :as-alias http.debug]
[app.http.management :as mgmt]
[app.http.session :as-alias session]
[app.http.session.tasks :as-alias session.tasks]
[app.http.websocket :as http.ws]
@@ -40,7 +38,7 @@
[app.storage.gc-touched :as-alias sto.gc-touched]
[app.storage.s3 :as-alias sto.s3]
[app.svgo :as-alias svgo]
[app.util.cron]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.test :as test]
[clojure.tools.namespace.repl :as repl]
@@ -233,8 +231,7 @@
::http/router (ig/ref ::http/router)
::http/io-threads (cf/get :http-server-io-threads)
::http/max-body-size (cf/get :http-server-max-body-size)
::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)
::wrk/executor (ig/ref ::wrk/executor)}
::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
::ldap/provider
{:host (cf/get :ldap-host)
@@ -274,10 +271,6 @@
::email/blacklist (ig/ref ::email/blacklist)
::email/whitelist (ig/ref ::email/whitelist)}
::mgmt/routes
{::db/pool (ig/ref ::db/pool)
::setup/props (ig/ref ::setup/props)}
:app.http/router
{::session/manager (ig/ref ::session/manager)
::db/pool (ig/ref ::db/pool)
@@ -286,7 +279,6 @@
::setup/props (ig/ref ::setup/props)
::mtx/routes (ig/ref ::mtx/routes)
::oidc/routes (ig/ref ::oidc/routes)
::mgmt/routes (ig/ref ::mgmt/routes)
::http.debug/routes (ig/ref ::http.debug/routes)
::http.assets/routes (ig/ref ::http.assets/routes)
::http.ws/routes (ig/ref ::http.ws/routes)
@@ -306,8 +298,8 @@
:app.http.assets/routes
{::http.assets/path (cf/get :assets-path)
::http.assets/cache-max-age (ct/duration {:hours 24})
::http.assets/cache-max-agesignature-max-age (ct/duration {:hours 24 :minutes 5})
::http.assets/cache-max-age (dt/duration {:hours 24})
::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
::sto/storage (ig/ref ::sto/storage)}
::rpc/climit
@@ -488,33 +480,33 @@
{::wrk/registry (ig/ref ::wrk/registry)
::db/pool (ig/ref ::db/pool)
::wrk/entries
[{:cron #penpot/cron "0 0 0 * * ?" ;; daily
[{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :session-gc}
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :objects-gc}
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :storage-gc-deleted}
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :storage-gc-touched}
{:cron #penpot/cron "0 0 0 * * ?" ;; daily
{:cron #app/cron "0 0 0 * * ?" ;; daily
:task :tasks-gc}
{:cron #penpot/cron "0 0 2 * * ?" ;; daily
{:cron #app/cron "0 0 2 * * ?" ;; daily
:task :file-gc-scheduler}
{:cron #penpot/cron "0 30 */3,23 * * ?"
{:cron #app/cron "0 30 */3,23 * * ?"
:task :telemetry}
(when (contains? cf/flags :audit-log-archive)
{:cron #penpot/cron "0 */5 * * * ?" ;; every 5m
{:cron #app/cron "0 */5 * * * ?" ;; every 5m
:task :audit-log-archive})
(when (contains? cf/flags :audit-log-gc)
{:cron #penpot/cron "30 */5 * * * ?" ;; every 5m
{:cron #app/cron "30 */5 * * * ?" ;; every 5m
:task :audit-log-gc})]}
::wrk/dispatcher

View File

@@ -14,11 +14,11 @@
[app.common.media :as cm]
[app.common.schema :as sm]
[app.common.schema.openapi :as-alias oapi]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as-alias db]
[app.storage :as-alias sto]
[app.storage.tmp :as tmp]
[app.util.time :as dt]
[buddy.core.bytes :as bb]
[buddy.core.codecs :as bc]
[clojure.java.shell :as sh]
@@ -38,13 +38,15 @@
org.im4java.core.Info))
(def schema:upload
[:map {:title "Upload"}
[:filename :string]
[:size ::sm/int]
[:path ::fs/path]
[:mtype {:optional true} :string]
[:headers {:optional true}
[:map-of :string :string]]])
(sm/register!
^{::sm/type ::upload}
[:map {:title "Upload"}
[:filename :string]
[:size ::sm/int]
[:path ::fs/path]
[:mtype {:optional true} :string]
[:headers {:optional true}
[:map-of :string :string]]]))
(def ^:private schema:input
[:map {:title "Input"}
@@ -116,7 +118,7 @@
(defn- parse-svg
[text]
(let [text (strip-doctype text)]
(dm/with-open [istream (IOUtils/toInputStream ^String text "UTF-8")]
(dm/with-open [istream (IOUtils/toInputStream text "UTF-8")]
(xml/parse istream secure-parser-factory))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -241,7 +243,7 @@
(ex/raise :type :validation
:code :invalid-svg-file
:hint "uploaded svg does not provides dimensions"))
(merge input info {:ts (ct/now)}))
(merge input info {:ts (dt/now)}))
(let [instance (Info. (str path))
mtype' (.getProperty instance "Mime type")]
@@ -261,7 +263,7 @@
(assoc input
:width width
:height height
:ts (ct/now)))))))
:ts (dt/now)))))))
(defmethod process-error org.im4java.core.InfoException
[error]

View File

@@ -438,16 +438,7 @@
:fn (mg/resource "app/migrations/sql/0138-mod-file-data-fragment-table.sql")}
{:name "0139-mod-file-change-table.sql"
:fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}
{:name "0140-mod-file-change-table.sql"
:fn (mg/resource "app/migrations/sql/0140-mod-file-change-table.sql")}
{:name "0140-add-locked-by-column-to-file-change-table"
:fn (mg/resource "app/migrations/sql/0140-add-locked-by-column-to-file-change-table.sql")}
{:name "0141-add-idx-to-file-library-rel"
:fn (mg/resource "app/migrations/sql/0141-add-idx-to-file-library-rel.sql")}])
:fn (mg/resource "app/migrations/sql/0139-mod-file-change-table.sql")}])
(defn apply-migrations!
[pool name migrations]

View File

@@ -1,11 +0,0 @@
-- Add locked_by column to file_change table for version locking feature
-- This allows users to lock their own saved versions to prevent deletion by others
ALTER TABLE file_change
ADD COLUMN locked_by uuid NULL REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE;
-- Create index for locked versions queries
CREATE INDEX file_change__locked_by__idx ON file_change (locked_by) WHERE locked_by IS NOT NULL;
-- Add comment for documentation
COMMENT ON COLUMN file_change.locked_by IS 'Profile ID of user who has locked this version. Only the creator can lock/unlock their own versions. Locked versions cannot be deleted by others.';

View File

@@ -1,2 +0,0 @@
ALTER TABLE file_change
ADD COLUMN migrations text[];

View File

@@ -1,2 +0,0 @@
CREATE INDEX IF NOT EXISTS file_library_rel__library_file_id__idx
ON file_library_rel (library_file_id);

View File

@@ -10,10 +10,10 @@
[app.common.data :as d]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.transit :as t]
[app.config :as cfg]
[app.redis :as rds]
[app.util.time :as dt]
[app.worker :as wrk]
[integrant.core :as ig]
[promesa.core :as p]
@@ -56,7 +56,7 @@
[k v]
{k (-> (d/without-nils v)
(assoc ::buffer-size 128)
(assoc ::timeout (ct/duration {:seconds 30})))})
(assoc ::timeout (dt/duration {:seconds 30})))})
(def ^:private schema:params
[:map ::rds/redis ::wrk/executor])

View File

@@ -12,10 +12,10 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.metrics :as mtx]
[app.redis.script :as-alias rscript]
[app.util.cache :as cache]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.core :as c]
[clojure.java.io :as io]
@@ -114,7 +114,7 @@
(let [cpus (px/get-available-processors)
threads (max 1 (int (* cpus 0.2)))]
{k (-> (d/without-nils v)
(assoc ::timeout (ct/duration "10s"))
(assoc ::timeout (dt/duration "10s"))
(assoc ::io-threads (max 3 threads))
(assoc ::worker-threads (max 3 threads)))}))
@@ -125,7 +125,7 @@
[::uri ::sm/uri]
[::worker-threads ::sm/int]
[::io-threads ::sm/int]
[::timeout ::ct/duration]])
[::timeout ::dt/duration]])
(defmethod ig/assert-key ::redis
[_ params]
@@ -331,7 +331,7 @@
(p/rejected cause))))
(eval-script [sha]
(let [tpoint (ct/tpoint)]
(let [tpoint (dt/tpoint)]
(->> (.evalsha ^RedisScriptingAsyncCommands cmd
^String sha
^ScriptOutputType ScriptOutputType/MULTI
@@ -346,7 +346,7 @@
:name (name sname)
:sha sha
:params (str/join "," (::rscript/vals script))
:elapsed (ct/format-duration elapsed))
:elapsed (dt/format-duration elapsed))
result)))
(p/merr on-error))))

View File

@@ -12,7 +12,6 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.spec :as us]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.http :as-alias http]
@@ -32,6 +31,7 @@
[app.storage :as-alias sto]
[app.util.inet :as inet]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]
@@ -103,7 +103,7 @@
data (-> params
(assoc ::handler-name handler-name)
(assoc ::ip-addr ip-addr)
(assoc ::request-at (ct/now))
(assoc ::request-at (dt/now))
(assoc ::external-session-id session-id)
(assoc ::external-event-origin event-origin)
(assoc ::session/id (::session/id request))
@@ -130,7 +130,7 @@
[{:keys [::mtx/metrics ::metrics-id]} f mdata]
(let [labels (into-array String [(::sv/name mdata)])]
(fn [cfg params]
(let [tp (ct/tpoint)]
(let [tp (dt/tpoint)]
(try
(f cfg params)
(finally

View File

@@ -11,11 +11,11 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.metrics :as mtx]
[app.rpc :as-alias rpc]
[app.util.cache :as cache]
[app.util.services :as-alias sv]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.edn :as edn]
[clojure.set :as set]
@@ -154,7 +154,7 @@
:id limit-id
:label limit-label
:queue queue
:elapsed (some-> elapsed ct/format-duration)
:elapsed (some-> elapsed dt/format-duration)
:params @limit-params)))
(def ^:private idseq (AtomicLong. 0))
@@ -171,19 +171,19 @@
mlabels (into-array String [(id->str limit-id)])
limit-id (id->str limit-id limit-key)
limiter (cache/get cache limit-id (partial create-limiter config))
tpoint (ct/tpoint)
tpoint (dt/tpoint)
req-id (.incrementAndGet ^AtomicLong idseq)]
(try
(let [stats (pbh/get-stats limiter)]
(measure metrics mlabels stats nil)
(log "enqueued" req-id stats limit-id limit-label limit-params nil))
(pbh/invoke! limiter (fn []
(let [elapsed (tpoint)
stats (pbh/get-stats limiter)]
(measure metrics mlabels stats elapsed)
(log "acquired" req-id stats limit-id limit-label limit-params elapsed)
(handler))))
(px/invoke! limiter (fn []
(let [elapsed (tpoint)
stats (pbh/get-stats limiter)]
(measure metrics mlabels stats elapsed)
(log "acquired" req-id stats limit-id limit-label limit-params elapsed)
(handler))))
(catch ExceptionInfo cause
(let [{:keys [type code]} (ex-data cause)]

View File

@@ -7,7 +7,6 @@
(ns app.rpc.commands.access-token
(:require
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.db :as db]
[app.main :as-alias main]
@@ -16,7 +15,8 @@
[app.rpc.quotes :as quotes]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.util.services :as sv]))
[app.util.services :as sv]
[app.util.time :as dt]))
(defn- decode-row
[row]
@@ -24,13 +24,13 @@
(defn create-access-token
[{:keys [::db/conn ::setup/props]} profile-id name expiration]
(let [created-at (ct/now)
(let [created-at (dt/now)
token-id (uuid/next)
token (tokens/generate props {:iss "access-token"
:tid token-id
:iat created-at})
expires-at (some-> expiration ct/in-future)
expires-at (some-> expiration dt/in-future)
token (db/insert! conn :access-token
{:id token-id
:name name
@@ -49,7 +49,7 @@
(def ^:private schema:create-access-token
[:map {:title "create-access-token"}
[:name [:string {:max 250 :min 1}]]
[:expiration {:optional true} ::ct/duration]])
[:expiration {:optional true} ::dt/duration]])
(sv/defmethod ::create-access-token
{::doc/added "1.18"

View File

@@ -10,7 +10,6 @@
[app.common.data :as d]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -21,7 +20,8 @@
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.util.inet :as inet]
[app.util.services :as sv]))
[app.util.services :as sv]
[app.util.time :as dt]))
(def ^:private event-columns
[:id
@@ -49,7 +49,7 @@
(defn- adjust-timestamp
[{:keys [timestamp created-at] :as event}]
(let [margin (inst-ms (ct/diff timestamp created-at))]
(let [margin (inst-ms (dt/diff timestamp created-at))]
(if (or (neg? margin)
(> margin 3600000))
;; If event is in future or lags more than 1 hour, we reasign
@@ -63,7 +63,7 @@
[{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}]
(let [request (-> params meta ::http/request)
ip-addr (inet/parse-request request)
tnow (ct/now)
tnow (dt/now)
xform (comp
(map (fn [event]
(-> event

View File

@@ -12,7 +12,6 @@
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -31,6 +30,7 @@
[app.setup.welcome-file :refer [create-welcome-file]]
[app.tokens :as tokens]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]))
@@ -42,7 +42,7 @@
(defn- elapsed-verify-threshold?
[profile]
(let [elapsed (ct/diff (:modified-at profile) (ct/now))
(let [elapsed (dt/diff (:modified-at profile) (dt/now))
verify-threshold (cf/get :email-verify-threshold)]
(pos? (compare elapsed verify-threshold))))
@@ -85,7 +85,7 @@
(ex/raise :type :validation
:code :wrong-credentials))
(when-let [deleted-at (:deleted-at profile)]
(when (ct/is-after? (ct/now) deleted-at)
(when (dt/is-after? (dt/now) deleted-at)
(ex/raise :type :validation
:code :wrong-credentials)))
@@ -244,7 +244,7 @@
:backend "penpot"
:iss :prepared-register
:profile-id (:id profile)
:exp (ct/in-future {:days 7})
:exp (dt/in-future {:days 7})
:props {:newsletter-updates (or accept-newsletter-updates false)}}
params (d/without-nils params)
@@ -344,7 +344,7 @@
[{:keys [::db/conn] :as cfg} profile]
(let [vtoken (tokens/generate (::setup/props cfg)
{:iss :verify-email
:exp (ct/in-future "72h")
:exp (dt/in-future "72h")
:profile-id (:id profile)
:email (:email profile)})
;; NOTE: this token is mainly used for possible complains
@@ -352,7 +352,7 @@
ptoken (tokens/generate (::setup/props cfg)
{:iss :profile-identity
:profile-id (:id profile)
:exp (ct/in-future {:days 30})})]
:exp (dt/in-future {:days 30})})]
(eml/send! {::eml/conn conn
::eml/factory eml/register
:public-uri (cf/get :public-uri)
@@ -466,7 +466,7 @@
(when (= action "resend-email-verification")
(db/update! conn :profile
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id (:id profile)})
(send-email-verification! cfg profile))
@@ -495,7 +495,7 @@
(letfn [(create-recovery-token [{:keys [id] :as profile}]
(let [token (tokens/generate (::setup/props cfg)
{:iss :password-recovery
:exp (ct/in-future "15m")
:exp (dt/in-future "15m")
:profile-id id})]
(assoc profile :token token)))
@@ -503,7 +503,7 @@
(let [ptoken (tokens/generate (::setup/props cfg)
{:iss :profile-identity
:profile-id (:id profile)
:exp (ct/in-future {:days 30})})]
:exp (dt/in-future {:days 30})})]
(eml/send! {::eml/conn conn
::eml/factory eml/password-recovery
:public-uri (cf/get :public-uri)
@@ -544,7 +544,7 @@
:else
(do
(db/update! conn :profile
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id (:id profile)})
(->> profile
(create-recovery-token)

View File

@@ -13,7 +13,6 @@
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.http.sse :as sse]
@@ -27,6 +26,7 @@
[app.rpc.doc :as-alias doc]
[app.tasks.file-gc]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[promesa.exec :as px]
[yetti.response :as yres]))
@@ -114,7 +114,7 @@
3 (px/invoke! executor (partial bf.v3/import-files! cfg)))]
(db/update! pool :project
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id project-id}
{::db/return-keys false})
@@ -125,35 +125,21 @@
[:name [:or [:string {:max 250}]
[:map-of ::sm/uuid [:string {:max 250}]]]]
[:project-id ::sm/uuid]
[:file-id {:optional true} ::sm/uuid]
[:version {:optional true} ::sm/int]
[:file media/schema:upload]])
[:file ::media/upload]])
(sv/defmethod ::import-binfile
"Import a penpot file in a binary format. If `file-id` is provided,
an in-place import will be performed instead of creating a new file.
The in-place imports are only supported for binfile-v3 and when a
.penpot file only contains one penpot file.
"
"Import a penpot file in a binary format."
{::doc/added "1.15"
::doc/changes ["1.20" "Add file-id param for in-place import"
"1.20" "Set default version to 3"]
::webhooks/event? true
::sse/stream? true
::sm/params schema:import-binfile}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}]
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}]
(projects/check-edition-permissions! pool profile-id project-id)
(let [version (or version 3)
(let [version (or version 1)
params (-> params
(assoc :profile-id profile-id)
(assoc :version version))
cfg (cond-> cfg
(uuid? file-id)
(assoc ::bfc/file-id file-id))
manifest (case (int version)
1 nil
3 (bf.v3/get-manifest (:path file)))]
@@ -161,6 +147,5 @@
(with-meta
(sse/response (partial import-binfile cfg params))
{::audit/props {:file nil
:file-id file-id
:generated-by (:generated-by manifest)
:referer (:referer manifest)}})))

View File

@@ -11,7 +11,6 @@
[app.common.exceptions :as ex]
[app.common.geom.point :as gpt]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as uri]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -30,6 +29,7 @@
[app.rpc.retry :as rtry]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.set :as set]
[cuerdas.core :as str]))
@@ -222,7 +222,7 @@
(defn upsert-comment-thread-status!
([conn profile-id thread-id]
(upsert-comment-thread-status! conn profile-id thread-id (ct/in-future "1s")))
(upsert-comment-thread-status! conn profile-id thread-id (dt/in-future "1s")))
([conn profile-id thread-id mod-at]
(db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id mod-at mod-at])))

View File

@@ -8,7 +8,6 @@
"A demo specific mutations."
(:require
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.loggers.audit :as audit]
@@ -17,6 +16,7 @@
[app.rpc.commands.profile :as profile]
[app.rpc.doc :as-alias doc]
[app.util.services :as sv]
[app.util.time :as dt]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]))
@@ -45,13 +45,15 @@
params {:email email
:fullname fullname
:is-active true
:deleted-at (ct/in-future (cf/get-deletion-delay))
:deleted-at (dt/in-future (cf/get-deletion-delay))
:password (profile/derive-password cfg password)
:props {}}
profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
(->> (auth/create-profile! conn params)
(auth/create-profile-rels! conn))))]
(with-meta {:email email
:password password}
{::audit/profile-id (:id profile)})))
:props {}}]
(let [profile (db/tx-run! cfg (fn [{:keys [::db/conn]}]
(->> (auth/create-profile! conn params)
(auth/create-profile-rels! conn))))]
(with-meta {:email email
:password password}
{::audit/profile-id (:id profile)}))))

View File

@@ -16,7 +16,6 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.schema.desc-js-like :as-alias smdj]
[app.common.time :as ct]
[app.common.types.components-list :as ctkl]
[app.common.types.file :as ctf]
[app.common.uri :as uri]
@@ -38,6 +37,7 @@
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]
[promesa.exec :as px]))
@@ -52,7 +52,7 @@
;; --- HELPERS
(def long-cache-duration
(ct/duration {:days 7}))
(dt/duration {:days 7}))
(defn decode-row
[{:keys [data changes features] :as row}]
@@ -78,7 +78,6 @@
;; --- FILE PERMISSIONS
(def ^:private sql:file-permissions
"select fpr.is_owner,
fpr.is_admin,
@@ -188,15 +187,15 @@
[:name [:string {:max 250}]]
[:revn [::sm/int {:min 0}]]
[:vern [::sm/int {:min 0}]]
[:modified-at ::ct/inst]
[:modified-at ::dt/instant]
[:is-shared ::sm/boolean]
[:project-id ::sm/uuid]
[:created-at ::ct/inst]
[:created-at ::dt/instant]
[:data {:optional true} ::sm/any]])
(def schema:permissions-mixin
[:map {:title "PermissionsMixin"}
[:permissions perms/schema:permissions]])
[:permissions ::perms/permissions]])
(def schema:file-with-permissions
[:merge {:title "FileWithPermissions"}
@@ -305,7 +304,7 @@
(defn get-file-etag
[{:keys [::rpc/profile-id]} {:keys [modified-at revn vern permissions]}]
(str profile-id "/" revn "/" vern "/" (hash fmg/available-migrations) "/"
(ct/format-inst modified-at :iso)
(dt/format-instant modified-at :iso)
"/"
(uri/map->query-string permissions)))
@@ -342,24 +341,14 @@
(cfeat/check-client-features! (:features params))
(cfeat/check-file-features! (:features file)))
(as-> file file
;; This operation is needed for backward comapatibility with
;; frontends that does not support pointer-map resolution
;; mechanism; this just resolves the pointers on backend and
;; return a complete file
(if (and (contains? (:features file) "fdata/pointer-map")
(not (contains? (:features params) "fdata/pointer-map")))
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(update file :data feat.fdata/process-pointers deref))
file)
;; This operation is needed for backward comapatibility with
;; frontends that does not support objects-map mechanism; this
;; just converts all objects map instaces to plain maps
(if (and (contains? (:features file) "fdata/objects-map")
(not (contains? (:features params) "fdata/objects-map")))
(update file :data feat.fdata/process-objects (partial into {}))
file)))))
;; This operation is needed for backward comapatibility with frontends that
;; does not support pointer-map resolution mechanism; this just resolves the
;; pointers on backend and return a complete file.
(if (and (contains? (:features file) "fdata/pointer-map")
(not (contains? (:features params) "fdata/pointer-map")))
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(update file :data feat.fdata/process-pointers deref))
file))))
;; --- COMMAND QUERY: get-file-fragment (by id)
@@ -367,8 +356,8 @@
[:map {:title "FileFragment"}
[:id ::sm/uuid]
[:file-id ::sm/uuid]
[:created-at ::ct/inst]
[:content ::sm/any]])
[:created-at ::dt/instant]
[:content any?]])
(def schema:get-file-fragment
[:map {:title "get-file-fragment"}
@@ -471,42 +460,8 @@
(:has-libraries row)))
;; --- COMMAND QUERY: get-library-usage
(declare get-library-usage)
(def schema:get-library-usage
[:map {:title "get-library-usage"}
[:file-id ::sm/uuid]])
:sample
(sv/defmethod ::get-library-usage
"Gets the number of files that use the specified library."
{::doc/added "2.10.0"
::sm/params schema:get-library-usage
::sm/result ::sm/int}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id]}]
(dm/with-open [conn (db/open pool)]
(check-read-permissions! pool profile-id file-id)
(get-library-usage conn file-id)))
(def ^:private sql:get-library-usage
"SELECT COUNT(*) AS used
FROM file_library_rel AS flr
JOIN file AS fl ON (flr.library_file_id = fl.id)
WHERE flr.library_file_id = ?::uuid
AND (fl.deleted_at IS NULL OR
fl.deleted_at > now())")
(defn- get-library-usage
[conn file-id]
(let [row (db/exec-one! conn [sql:get-library-usage file-id])]
{:used-in (:used row)}))
;; --- QUERY COMMAND: get-page
(defn- prune-objects
"Given the page data and the object-id returns the page data with all
other not needed objects removed from the `:objects` data
@@ -596,24 +551,6 @@
;; --- COMMAND QUERY: get-team-shared-files
(defn- components-and-variants
"Return a set with all the variant-ids, and a list of components, but with
only one component by variant"
[components]
(let [{:keys [variant-ids components]}
(reduce (fn [{:keys [variant-ids components] :as acc} {:keys [variant-id] :as component}]
(cond
(nil? variant-id)
{:variant-ids variant-ids :components (conj components component)}
(contains? variant-ids variant-id)
acc
:else
{:variant-ids (conj variant-ids variant-id) :components (conj components component)}))
{:variant-ids #{} :components []}
components)]
{:components components
:variant-ids variant-ids}))
(def ^:private sql:team-shared-files
"select f.id,
f.revn,
@@ -647,13 +584,10 @@
:sample (into [] (take limit sorted-assets))}))]
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(let [load-objects (fn [component]
(ctf/load-component-objects data component))
comps-and-variants (components-and-variants (ctkl/components-seq data))
components (into {} (map (juxt :id identity) (:components comps-and-variants)))
components-sample (-> (assets-sample components 4)
(update :sample #(mapv load-objects %))
(assoc :variants-count (-> comps-and-variants :variant-ids count)))]
(let [load-objects (fn [component]
(ctf/load-component-objects data component))
components-sample (-> (assets-sample (ctkl/components data) 4)
(update :sample #(mapv load-objects %)))]
{:components components-sample
:media (assets-sample (:media data) 3)
:colors (assets-sample (:colors data) 3)
@@ -707,7 +641,6 @@
;; --- COMMAND QUERY: Files that use this File library
(def ^:private sql:library-using-files
"SELECT f.id,
f.name
@@ -780,7 +713,6 @@
;; --- COMMAND QUERY: get-file-summary
(defn- get-file-summary
[{:keys [::db/conn] :as cfg} {:keys [profile-id id project-id] :as params}]
(check-read-permissions! conn profile-id id)
@@ -798,13 +730,11 @@
(cfeat/check-file-features! (:features file)))
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
(let [components-and-variants (components-and-variants (ctkl/components-seq (:data file)))]
{:name (:name file)
:components-count (-> components-and-variants :components count)
:variants-count (-> components-and-variants :variant-ids count)
:graphics-count (count (get-in file [:data :media] []))
:colors-count (count (get-in file [:data :colors] []))
:typography-count (count (get-in file [:data :typographies] []))}))))
{:name (:name file)
:components-count (count (ctkl/components-seq (:data file)))
:graphics-count (count (get-in file [:data :media] []))
:colors-count (count (get-in file [:data :colors] []))
:typography-count (count (get-in file [:data :typographies] []))})))
(sv/defmethod ::get-file-summary
"Retrieve a file summary by its ID. Only authenticated users."
@@ -816,7 +746,6 @@
;; --- COMMAND QUERY: get-file-info
(defn- get-file-info
[{:keys [::db/conn] :as cfg} {:keys [id] :as params}]
(db/get* conn :file
@@ -841,7 +770,7 @@
[conn {:keys [id name]}]
(db/update! conn :file
{:name name
:modified-at (ct/now)}
:modified-at (dt/now)}
{:id id}
{::db/return-keys true}))
@@ -854,8 +783,8 @@
[:id ::sm/uuid]
[:project-id ::sm/uuid]
[:name [:string {:max 250}]]
[:created-at ::ct/inst]
[:modified-at ::ct/inst]]
[:created-at ::dt/instant]
[:modified-at ::dt/instant]]
::sm/params
[:map {:title "RenameFileParams"}
@@ -866,8 +795,8 @@
[:map {:title "SimplifiedFile"}
[:id ::sm/uuid]
[:name [:string {:max 250}]]
[:created-at ::ct/inst]
[:modified-at ::ct/inst]]
[:created-at ::dt/instant]
[:modified-at ::dt/instant]]
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id id] :as params}]
@@ -910,7 +839,7 @@
(db/update! cfg :file
{:revn (inc (:revn file))
:data (blob/encode (:data file))
:modified-at (ct/now)
:modified-at (dt/now)
:has-media-trimmed false}
{:id file-id})
@@ -971,7 +900,7 @@
(db/delete! conn :file-library-rel {:library-file-id id})
(db/update! conn :file
{:is-shared false
:modified-at (ct/now)}
:modified-at (dt/now)}
{:id id})
(select-keys file [:id :name :is-shared]))
@@ -980,7 +909,7 @@
(let [file (assoc file :is-shared true)]
(db/update! conn :file
{:is-shared true
:modified-at (ct/now)}
:modified-at (dt/now)}
{:id id})
file)
@@ -1016,7 +945,7 @@
[conn team file-id]
(let [delay (ldel/get-deletion-delay team)
file (db/update! conn :file
{:deleted-at (ct/in-future delay)}
{:deleted-at (dt/in-future delay)}
{:id file-id}
{::db/return-keys [:id :name :is-shared :deleted-at
:project-id :created-at :modified-at]})]
@@ -1114,7 +1043,7 @@
(defn update-sync
[conn {:keys [file-id library-id] :as params}]
(db/update! conn :file-library-rel
{:synced-at (ct/now)}
{:synced-at (dt/now)}
{:file-id file-id
:library-file-id library-id}
{::db/return-keys true}))
@@ -1139,14 +1068,14 @@
[conn {:keys [file-id date] :as params}]
(db/update! conn :file
{:ignore-sync-until date
:modified-at (ct/now)}
:modified-at (dt/now)}
{:id file-id}
{::db/return-keys true}))
(def ^:private schema:ignore-file-library-sync-status
[:map {:title "ignore-file-library-sync-status"}
[:file-id ::sm/uuid]
[:date ::ct/inst]])
[:date ::dt/instant]])
;; TODO: improve naming
(sv/defmethod ::ignore-file-library-sync-status

View File

@@ -9,7 +9,6 @@
[app.binfile.common :as bfc]
[app.common.features :as cfeat]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.file :as ctf]
[app.config :as cf]
[app.db :as db]
@@ -23,13 +22,13 @@
[app.rpc.quotes :as quotes]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.set :as set]))
(defn create-file-role!
[conn {:keys [file-id profile-id role]}]
(let [params {:file-id file-id
:profile-id profile-id}]
(->> (perms/assign-role-flags params role)
(db/insert! conn :file-profile-rel))))
@@ -52,18 +51,18 @@
:is-shared is-shared
:features features
:ignore-sync-until ignore-sync-until
:created-at modified-at
:modified-at modified-at
:deleted-at deleted-at}
{:create-page create-page
:page-id page-id})]
(bfc/insert-file! cfg file)
:page-id page-id})
file (-> (bfc/insert-file! cfg file)
(bfc/decode-row))]
(->> (assoc params :file-id (:id file) :role :owner)
(create-file-role! conn))
(db/update! conn :project
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id project-id})
file)))
@@ -112,15 +111,14 @@
;; FIXME: IMPORTANT: this code can have race conditions, because
;; we have no locks for updating team so, creating two files
;; concurrently can lead to lost team features updating
(when-let [features (-> features
(set/difference (:features team))
(set/difference cfeat/no-team-inheritable-features)
(not-empty))]
(let [features (-> features
(set/union (:features team))
(set/difference cfeat/no-team-inheritable-features)
(into-array))]
(let [features (->> features
(set/union (:features team))
(db/create-array conn "text"))]
(db/update! conn :team
{:features features}
{:id (:id team)}

View File

@@ -8,16 +8,13 @@
(:require
[app.binfile.common :as bfc]
[app.common.exceptions :as ex]
[app.common.files.migrations :as fmg]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.fdata :as feat.fdata]
[app.features.file-migrations :refer [reset-migrations!]]
[app.main :as-alias main]
[app.msgbus :as mbus]
[app.rpc :as-alias rpc]
@@ -27,18 +24,12 @@
[app.storage :as sto]
[app.util.blob :as blob]
[app.util.services :as sv]
[app.util.time :as dt]
[cuerdas.core :as str]))
(defn decode-row
[{:keys [migrations] :as row}]
(when row
(cond-> row
(some? migrations)
(assoc :migrations (db/decode-pgarray migrations)))))
(def sql:get-file-snapshots
"WITH changes AS (
SELECT id, label, revn, created_at, created_by, profile_id, locked_by
SELECT id, label, revn, created_at, created_by, profile_id
FROM file_change
WHERE file_id = ?
AND data IS NOT NULL
@@ -69,8 +60,8 @@
(defn- generate-snapshot-label
[]
(let [ts (-> (ct/now)
(ct/format-inst)
(let [ts (-> (dt/now)
(dt/format-instant)
(str/replace #"[T:\.]" "-")
(str/rtrim "Z"))]
(str "snapshot-" ts)))
@@ -83,15 +74,18 @@
(assert (#{:system :user :admin} created-by)
"expected valid keyword for created-by")
(let [created-by
(let [conn
(db/get-connection cfg)
created-by
(name created-by)
deleted-at
(cond
(= deleted-at :default)
(ct/plus (ct/now) (cf/get-deletion-delay))
(dt/plus (dt/now) (cf/get-deletion-delay))
(ct/inst? deleted-at)
(dt/instant? deleted-at)
deleted-at
:else
@@ -107,15 +101,12 @@
(blob/encode (:data file))
features
(into-array (:features file))
(db/encode-pgarray (:features file) conn "text")]
migrations
(into-array (:migrations file))]
(l/dbg :hint "creating file snapshot"
:file-id (str (:id file))
:id (str snapshot-id)
:label label)
(l/debug :hint "creating file snapshot"
:file-id (str (:id file))
:id (str snapshot-id)
:label label)
(db/insert! cfg :file-change
{:id snapshot-id
@@ -123,7 +114,6 @@
:data data
:version (:version file)
:features features
:migrations migrations
:profile-id profile-id
:file-id (:id file)
:label label
@@ -169,17 +159,7 @@
{:file-id file-id
:id snapshot-id}
{::db/for-share true})
(feat.fdata/resolve-file-data cfg)
(decode-row))
;; If snapshot has tracked applied migrations, we reuse them,
;; if not we take a safest set of migrations as starting
;; point. This is because, at the time of implementing
;; snapshots, migrations were not taken into account so we
;; need to make this backward compatible in some way.
file (assoc file :migrations
(or (:migrations snapshot)
(fmg/generate-migrations-from-version 67)))]
(feat.fdata/resolve-file-data cfg))]
(when-not snapshot
(ex/raise :type :not-found
@@ -200,16 +180,12 @@
:label (:label snapshot)
:snapshot-id (str (:id snapshot)))
;; If the file was already offloaded, on restoring the snapshot we
;; are going to replace the file data, so we need to touch the old
;; referenced storage object and avoid possible leaks
;; If the file was already offloaded, on restring the snapshot
;; we are going to replace the file data, so we need to touch
;; the old referenced storage object and avoid possible leaks
(when (feat.fdata/offloaded? file)
(sto/touch-object! storage (:data-ref-id file)))
;; In the same way, on reseting the file data, we need to restore
;; the applied migrations on the moment of taking the snapshot
(reset-migrations! conn file)
(db/update! conn :file
{:data (:data snapshot)
:revn (inc (:revn file))
@@ -277,14 +253,14 @@
:deleted-at nil}
{:id snapshot-id}
{::db/return-keys true})
(dissoc :data :features :migrations)))
(dissoc :data :features)))
(defn- get-snapshot
"Get a minimal snapshot from database and lock for update"
[conn id]
(db/get conn :file-change
{:id id}
{::sql/columns [:id :file-id :created-by :deleted-at :profile-id :locked-by]
{::sql/columns [:id :file-id :created-by :deleted-at]
::db/for-update true}))
(sv/defmethod ::update-file-snapshot
@@ -304,7 +280,7 @@
(defn- delete-file-snapshot!
[conn snapshot-id]
(db/update! conn :file-change
{:deleted-at (ct/now)}
{:deleted-at (dt/now)}
{:id snapshot-id}
{::db/return-keys false})
nil)
@@ -324,111 +300,4 @@
:snapshot-id id
:profile-id profile-id))
;; Check if version is locked by someone else
(when (and (:locked-by snapshot)
(not= (:locked-by snapshot) profile-id))
(ex/raise :type :validation
:code :snapshot-is-locked
:hint "Cannot delete a locked version"
:snapshot-id id
:profile-id profile-id
:locked-by (:locked-by snapshot)))
(delete-file-snapshot! conn id)))))
;;; Lock/unlock version endpoints
(def ^:private schema:lock-file-snapshot
[:map {:title "lock-file-snapshot"}
[:id ::sm/uuid]])
(defn- lock-file-snapshot!
[conn snapshot-id profile-id]
(db/update! conn :file-change
{:locked-by profile-id}
{:id snapshot-id}
{::db/return-keys false})
nil)
(sv/defmethod ::lock-file-snapshot
{::doc/added "1.20"
::sm/params schema:lock-file-snapshot}
[cfg {:keys [::rpc/profile-id id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn]}]
(let [snapshot (get-snapshot conn id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
(ex/raise :type :validation
:code :system-snapshots-cant-be-locked
:hint "Only user-created versions can be locked"
:snapshot-id id
:profile-id profile-id))
;; Only the creator can lock their own version
(when (not= (:profile-id snapshot) profile-id)
(ex/raise :type :validation
:code :only-creator-can-lock
:hint "Only the version creator can lock it"
:snapshot-id id
:profile-id profile-id
:creator-id (:profile-id snapshot)))
;; Check if already locked
(when (:locked-by snapshot)
(ex/raise :type :validation
:code :snapshot-already-locked
:hint "Version is already locked"
:snapshot-id id
:profile-id profile-id
:locked-by (:locked-by snapshot)))
(lock-file-snapshot! conn id profile-id)))))
(def ^:private schema:unlock-file-snapshot
[:map {:title "unlock-file-snapshot"}
[:id ::sm/uuid]])
(defn- unlock-file-snapshot!
[conn snapshot-id]
(db/update! conn :file-change
{:locked-by nil}
{:id snapshot-id}
{::db/return-keys false})
nil)
(sv/defmethod ::unlock-file-snapshot
{::doc/added "1.20"
::sm/params schema:unlock-file-snapshot}
[cfg {:keys [::rpc/profile-id id]}]
(db/tx-run! cfg
(fn [{:keys [::db/conn]}]
(let [snapshot (get-snapshot conn id)]
(files/check-edition-permissions! conn profile-id (:file-id snapshot))
(when (not= (:created-by snapshot) "user")
(ex/raise :type :validation
:code :system-snapshots-cant-be-unlocked
:hint "Only user-created versions can be unlocked"
:snapshot-id id
:profile-id profile-id))
;; Only the creator can unlock their own version
(when (not= (:profile-id snapshot) profile-id)
(ex/raise :type :validation
:code :only-creator-can-unlock
:hint "Only the version creator can unlock it"
:snapshot-id id
:profile-id profile-id
:creator-id (:profile-id snapshot)))
;; Check if not locked
(when (not (:locked-by snapshot))
(ex/raise :type :validation
:code :snapshot-not-locked
:hint "Version is not locked"
:snapshot-id id
:profile-id profile-id))
(unlock-file-snapshot! conn id)))))

View File

@@ -10,7 +10,6 @@
[app.common.features :as cfeat]
[app.common.files.changes :as cpc]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -28,6 +27,7 @@
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.set :as set]))
;; --- MUTATION COMMAND: create-temp-file
@@ -72,16 +72,17 @@
params
(-> params
(assoc :profile-id profile-id)
(assoc :deleted-at (ct/in-future {:days 1}))
(assoc :deleted-at (dt/in-future {:days 1}))
(assoc :features features))]
(files.create/create-file cfg params)))
;; --- MUTATION COMMAND: update-temp-file
(def ^:private schema:update-temp-file
[:map {:title "update-temp-file"}
[:changes [:vector cpc/schema:change]]
[:changes [:vector ::cpc/change]]
[:revn [::sm/int {:min 0}]]
[:session-id ::sm/uuid]
[:id ::sm/uuid]])
@@ -96,7 +97,7 @@
{:id (uuid/next)
:session-id session-id
:profile-id profile-id
:created-at (ct/now)
:created-at (dt/now)
:file-id id
:revn revn
:data nil

View File

@@ -13,7 +13,6 @@
[app.common.geom.shapes :as gsh]
[app.common.schema :as sm]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.shape-tree :as ctt]
[app.config :as cf]
[app.db :as db]
@@ -31,12 +30,13 @@
[app.storage :as sto]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[cuerdas.core :as str]))
;; --- FEATURES
(def long-cache-duration
(ct/duration {:days 7}))
(dt/duration {:days 7}))
;; --- COMMAND QUERY: get-file-object-thumbnails
@@ -185,7 +185,7 @@
[:map {:title "PartialFile"}
[:id ::sm/uuid]
[:revn {:min 0} ::sm/int]
[:page [:map-of :keyword ::sm/any]]])
[:page :any]])
(sv/defmethod ::get-file-data-for-thumbnail
"Retrieves the data for generate the thumbnail of the file. Used
@@ -247,7 +247,7 @@
(defn- create-file-object-thumbnail!
[{:keys [::sto/storage] :as cfg} file object-id media tag]
(let [file-id (:id file)
timestamp (ct/now)
timestamp (dt/now)
media (persist-thumbnail! storage media timestamp)
[th1 th2] (db/tx-run! cfg (fn [{:keys [::db/conn]}]
(let [th1 (db/exec-one! conn [sql:get-file-object-thumbnail file-id object-id tag])
@@ -271,7 +271,7 @@
[:map {:title "create-file-object-thumbnail"}
[:file-id ::sm/uuid]
[:object-id [:string {:max 250}]]
[:media media/schema:upload]
[:media ::media/upload]
[:tag {:optional true} [:string {:max 50}]]])
(sv/defmethod ::create-file-object-thumbnail
@@ -302,7 +302,7 @@
{::sql/for-update true})]
(sto/touch-object! storage media-id)
(db/update! conn :file-tagged-object-thumbnail
{:deleted-at (ct/now)}
{:deleted-at (dt/now)}
{:file-id file-id
:object-id object-id
:tag tag})))
@@ -338,7 +338,7 @@
hash (sto/calculate-hash path)
data (-> (sto/content path)
(sto/wrap-with-hash hash))
tnow (ct/now)
tnow (dt/now)
media (sto/put-object! storage
{::sto/content data
::sto/deduplicate? true
@@ -381,7 +381,7 @@
[:map {:title "create-file-thumbnail"}
[:file-id ::sm/uuid]
[:revn ::sm/int]
[:media media/schema:upload]])
[:media ::media/upload]])
(sv/defmethod ::create-file-thumbnail
"Creates or updates the file thumbnail. Mainly used for paint the

View File

@@ -15,7 +15,6 @@
[app.common.files.validate :as val]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -37,6 +36,7 @@
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.set :as set]
[promesa.exec :as px]))
@@ -64,10 +64,10 @@
[:revn {:min 0} ::sm/int]
[:vern {:min 0} ::sm/int]
[:features {:optional true} ::cfeat/features]
[:changes {:optional true} [:vector cpc/schema:change]]
[:changes {:optional true} [:vector ::cpc/change]]
[:changes-with-metadata {:optional true}
[:vector [:map
[:changes [:vector cpc/schema:change]]
[:changes [:vector ::cpc/change]]
[:hint-origin {:optional true} :keyword]
[:hint-events {:optional true} [:vector [:string {:max 250}]]]]]]
[:skip-validate {:optional true} ::sm/boolean]])
@@ -76,7 +76,7 @@
schema:update-file-result
[:vector {:title "update-file-result"}
[:map
[:changes [:vector cpc/schema:change]]
[:changes [:vector ::cpc/change]]
[:file-id ::sm/uuid]
[:id ::sm/uuid]
[:revn {:min 0} ::sm/int]
@@ -123,7 +123,7 @@
[:update-file/global]]
::webhooks/event? true
::webhooks/batch-timeout (ct/duration "2m")
::webhooks/batch-timeout (dt/duration "2m")
::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
::sm/params schema:update-file
@@ -156,9 +156,10 @@
(assoc :file file)
(assoc :changes changes))
cfg (assoc cfg ::timestamp (ct/now))
cfg (assoc cfg ::timestamp (dt/now))
tpoint (dt/tpoint)]
tpoint (ct/tpoint)]
(when (not= (:vern params)
(:vern file))
@@ -182,15 +183,15 @@
(set/difference (:features team))
(set/difference cfeat/no-team-inheritable-features)
(not-empty))]
(let [features (-> features
(set/union (:features team))
(set/difference cfeat/no-team-inheritable-features)
(into-array))]
(let [features (->> features
(set/union (:features team))
(db/create-array conn "text"))]
(db/update! conn :team
{:features features}
{:id (:id team)}
{::db/return-keys false})))
(mtx/run! metrics {:id :update-file-changes :inc (count changes)})
(binding [l/*context* (some-> (meta params)
@@ -198,7 +199,7 @@
(errors/request->context))]
(-> (update-file* cfg params)
(rph/with-defer #(let [elapsed (tpoint)]
(l/trace :hint "update-file" :time (ct/format-duration elapsed))))))))))
(l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
(defn- update-file*
"Internal function, part of the update-file process, that encapsulates
@@ -243,8 +244,8 @@
:created-at timestamp
:updated-at timestamp
:deleted-at (if (::snapshot-data file)
(ct/plus timestamp (ldel/get-deletion-delay team))
(ct/plus timestamp (ct/duration {:hours 1})))
(dt/plus timestamp (ldel/get-deletion-delay team))
(dt/plus timestamp (dt/duration {:hours 1})))
:file-id (:id file)
:revn (:revn file)
:version (:version file)
@@ -305,7 +306,7 @@
[{:keys [::db/conn ::timestamp]} file]
(let [;; The timestamp can be nil because this function is also
;; intended to be used outside of this module
modified-at (or timestamp (ct/now))]
modified-at (or timestamp (dt/now))]
(db/update! conn :project
{:modified-at modified-at}
@@ -359,7 +360,7 @@
;; TODO: reuse operations if file is migrated
;; TODO: move encoding to a separated thread
file (if (take-snapshot? file)
(let [tpoint (ct/tpoint)
(let [tpoint (dt/tpoint)
snapshot (-> (:data file)
(feat.fdata/process-pointers deref)
(feat.fdata/process-objects (partial into {}))
@@ -371,7 +372,7 @@
:file-id (str (:id file))
:revn (:revn file)
:label label
:elapsed (ct/format-duration elapsed))
:elapsed (dt/format-duration elapsed))
(-> file
(assoc ::snapshot-data snapshot)
@@ -407,6 +408,7 @@
(not skip-validate))
(bfc/get-resolved-file-libraries cfg file))
;; The main purpose of this atom is provide a contextual state
;; for the changes subsystem where optionally some hints can
;; be provided for the changes processing. Right now we are
@@ -450,11 +452,11 @@
(when (contains? cf/flags :auto-file-snapshot)
(let [freq (or (cf/get :auto-file-snapshot-every) 20)
timeout (or (cf/get :auto-file-snapshot-timeout)
(ct/duration {:hours 1}))]
(dt/duration {:hours 1}))]
(or (= 1 freq)
(zero? (mod revn freq))
(> (inst-ms (ct/diff modified-at (ct/now)))
(> (inst-ms (dt/diff modified-at (dt/now)))
(inst-ms timeout))))))
(def ^:private sql:lagged-changes
@@ -494,5 +496,5 @@
:file-id (:id file)
:session-id session-id
:revn (:revn file)
:modified-at (ct/now)
:modified-at (dt/now)
:changes lchanges}))))

View File

@@ -9,7 +9,6 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.db :as db]
[app.db.sql :as-alias sql]
@@ -27,6 +26,7 @@
[app.rpc.quotes :as quotes]
[app.storage :as sto]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[promesa.exec :as px]))
@@ -37,13 +37,14 @@
(def ^:private
schema:get-font-variants
[:and
[:map {:title "get-font-variants"}
[:team-id {:optional true} ::sm/uuid]
[:file-id {:optional true} ::sm/uuid]
[:project-id {:optional true} ::sm/uuid]
[:share-id {:optional true} ::sm/uuid]]
[::sm/contains-any #{:team-id :file-id :project-id}]])
[:schema {:title "get-font-variants"}
[:and
[:map
[:team-id {:optional true} ::sm/uuid]
[:file-id {:optional true} ::sm/uuid]
[:project-id {:optional true} ::sm/uuid]
[:share-id {:optional true} ::sm/uuid]]
[::sm/contains-any #{:team-id :file-id :project-id}]]])
(sv/defmethod ::get-font-variants
{::doc/added "1.18"
@@ -123,7 +124,7 @@
content (-> (sto/content resource)
(sto/wrap-with-hash hash))]
{::sto/content content
::sto/touched-at (ct/now)
::sto/touched-at (dt/now)
::sto/deduplicate? true
:content-type mtype
:bucket "team-font-variant"})))
@@ -216,7 +217,7 @@
{::sql/for-update true})
delay (ldel/get-deletion-delay team)
tnow (ct/in-future delay)]
tnow (dt/in-future delay)]
(teams/check-edition-permissions! (:permissions team))
@@ -260,7 +261,7 @@
(teams/check-edition-permissions! (:permissions team))
(db/update! conn :team-font-variant
{:deleted-at (ct/in-future delay)}
{:deleted-at (dt/in-future delay)}
{:id (:id variant)}
{::db/return-keys false})

View File

@@ -13,7 +13,6 @@
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -29,6 +28,7 @@
[app.setup.templates :as tmpl]
[app.storage.tmp :as tmp]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[promesa.exec :as px]))
@@ -104,7 +104,7 @@
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
(binding [bfc/*state* (volatile! {:index {file-id (uuid/next)}})]
(duplicate-file (assoc cfg ::bfc/timestamp (ct/now))
(duplicate-file (assoc cfg ::bfc/timestamp (dt/now))
(-> params
(assoc :profile-id profile-id)
(assoc :reset-shared-flag true)))))))
@@ -164,7 +164,7 @@
(db/tx-run! cfg (fn [cfg]
;; Defer all constraints
(db/exec-one! cfg ["SET CONSTRAINTS ALL DEFERRED"])
(-> (assoc cfg ::bfc/timestamp (ct/now))
(-> (assoc cfg ::bfc/timestamp (dt/now))
(duplicate-project (assoc params :profile-id profile-id))))))
(defn duplicate-team
@@ -320,7 +320,7 @@
;; trully different modification date to each file.
(px/sleep 10)
(db/update! conn :project
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id project-id}))
nil))
@@ -425,7 +425,7 @@
(db/tx-run! cfg
(fn [{:keys [::db/conn] :as cfg}]
(db/update! conn :project
{:modified-at (ct/now)}
{:modified-at (dt/now)}
{:id project-id}
{::db/return-keys false})

View File

@@ -10,7 +10,6 @@
[app.common.exceptions :as ex]
[app.common.media :as cm]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -24,6 +23,7 @@
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[cuerdas.core :as str]
[datoteka.io :as io]
@@ -48,7 +48,7 @@
[:file-id ::sm/uuid]
[:is-local ::sm/boolean]
[:name [:string {:max 250}]]
[:content media/schema:upload]])
[:content ::media/upload]])
(sv/defmethod ::upload-file-media-object
{::doc/added "1.17"
@@ -67,7 +67,7 @@
mobj (create-file-media-object cfg params)]
(db/update! conn :file
{:modified-at (ct/now)
{:modified-at (dt/now)
:has-media-trimmed false}
{:id file-id}
{::db/return-keys false})
@@ -192,7 +192,7 @@
mobj (create-file-media-object-from-url cfg (assoc params :profile-id profile-id))]
(db/update! pool :file
{:modified-at (ct/now)
{:modified-at (dt/now)
:has-media-trimmed false}
{:id file-id}
{::db/return-keys false})

View File

@@ -10,7 +10,6 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.plugins :refer [schema:plugin-registry]]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -29,6 +28,7 @@
[app.storage :as sto]
[app.tokens :as tokens]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]
[promesa.exec :as px]))
@@ -70,8 +70,8 @@
[:is-blocked {:optional true} ::sm/boolean]
[:is-demo {:optional true} ::sm/boolean]
[:is-muted {:optional true} ::sm/boolean]
[:created-at {:optional true} ::ct/inst]
[:modified-at {:optional true} ::ct/inst]
[:created-at {:optional true} ::sm/inst]
[:modified-at {:optional true} ::sm/inst]
[:default-project-id {:optional true} ::sm/uuid]
[:default-team-id {:optional true} ::sm/uuid]
[:props {:optional true} schema:props]])
@@ -131,7 +131,9 @@
;; NOTE: we need to retrieve the profile independently if we use
;; it or not for explicit locking and avoid concurrent updates of
;; the same row/object.
(let [profile (get-profile conn profile-id ::db/for-update true)
(let [profile (-> (db/get-by-id conn :profile profile-id ::sql/for-update true)
(decode-row))
;; Update the profile map with direct params
profile (-> profile
(assoc :fullname fullname)
@@ -141,9 +143,9 @@
(db/update! conn :profile
{:fullname fullname
:lang lang
:theme theme}
{:id profile-id}
{::db/return-keys false})
:theme theme
:props (db/tjson (:props profile))}
{:id profile-id})
(-> profile
(strip-private-attrs)
@@ -226,22 +228,21 @@
(defn- update-notifications!
[{:keys [::db/conn] :as cfg} {:keys [profile-id dashboard-comments email-comments email-invites]}]
(let [profile
(get-profile conn profile-id ::db/for-update true)
(let [profile (get-profile conn profile-id)
notifications
{:dashboard-comments dashboard-comments
:email-comments email-comments
:email-invites email-invites}
:email-invites email-invites}]
props
(-> (get profile :props)
(assoc :notifications notifications))]
(db/update!
conn :profile
{:props
(-> (:props profile)
(assoc :notifications notifications)
(db/tjson))}
{:id (:id profile)})
(db/update! conn :profile
{:props (db/tjson props)}
{:id profile-id}
{::db/return-keys false})
nil))
;; --- MUTATION: Update Photo
@@ -252,7 +253,7 @@
(def ^:private
schema:update-profile-photo
[:map {:title "update-profile-photo"}
[:file media/schema:upload]])
[:file ::media/upload]])
(sv/defmethod ::update-profile-photo
{:doc/added "1.1"
@@ -351,13 +352,13 @@
[{:keys [::db/conn] :as cfg} {:keys [profile email] :as params}]
(let [token (tokens/generate (::setup/props cfg)
{:iss :change-email
:exp (ct/in-future "15m")
:exp (dt/in-future "15m")
:profile-id (:id profile)
:email email})
ptoken (tokens/generate (::setup/props cfg)
{:iss :profile-identity
:profile-id (:id profile)
:exp (ct/in-future {:days 30})})]
:exp (dt/in-future {:days 30})})]
(when (not= email (:email profile))
(check-profile-existence! conn params))
@@ -410,7 +411,7 @@
(defn update-profile-props
[{:keys [::db/conn] :as cfg} profile-id props]
(let [profile (get-profile conn profile-id ::db/for-update true)
(let [profile (get-profile conn profile-id ::sql/for-update true)
props (reduce-kv (fn [props k v]
;; We don't accept namespaced keys
(if (simple-ident? k)
@@ -423,17 +424,16 @@
(db/update! conn :profile
{:props (db/tjson props)}
{:id profile-id}
{::db/return-keys false})
{:id profile-id})
(filter-props props)))
(sv/defmethod ::update-profile-props
{::doc/added "1.0"
::sm/params schema:update-profile-props
::db/transaction true}
::sm/params schema:update-profile-props}
[cfg {:keys [::rpc/profile-id props]}]
(update-profile-props cfg profile-id props))
(db/tx-run! cfg (fn [cfg]
(update-profile-props cfg profile-id props))))
;; --- MUTATION: Delete Profile
@@ -444,7 +444,7 @@
::db/transaction true}
[{:keys [::db/conn] :as cfg} {:keys [::rpc/profile-id] :as params}]
(let [teams (get-owned-teams conn profile-id)
deleted-at (ct/now)]
deleted-at (dt/now)]
;; If we found owned teams with participants, we don't allow
;; delete profile until the user properly transfer ownership or
@@ -471,26 +471,6 @@
(-> (rph/wrap nil)
(rph/with-transform (session/delete-fn cfg)))))
(def sql:get-subscription-editors
"SELECT DISTINCT
p.id,
p.fullname AS name,
p.email AS email
FROM team_profile_rel AS tpr1
JOIN team_profile_rel AS tpr2
ON (tpr1.team_id = tpr2.team_id)
JOIN profile AS p
ON (tpr2.profile_id = p.id)
WHERE tpr1.profile_id = ?
AND tpr1.is_owner IS true
AND tpr2.can_edit IS true")
(sv/defmethod ::get-subscription-usage
{::doc/added "2.9"}
[cfg {:keys [::rpc/profile-id]}]
(let [editors (db/exec! cfg [sql:get-subscription-editors profile-id])]
{:editors editors}))
;; --- HELPERS
(def sql:owned-teams

View File

@@ -9,7 +9,6 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.features.logical-deletion :as ldel]
@@ -22,6 +21,7 @@
[app.rpc.permissions :as perms]
[app.rpc.quotes :as quotes]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]))
;; --- Check Project Permissions
@@ -218,7 +218,7 @@
(sv/defmethod ::update-project-pin
{::doc/added "1.18"
::sm/params schema:update-project-pin
::webhooks/batch-timeout (ct/duration "5s")
::webhooks/batch-timeout (dt/duration "5s")
::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
::webhooks/event? true
::db/transaction true}
@@ -257,7 +257,7 @@
[conn team project-id]
(let [delay (ldel/get-deletion-delay team)
project (db/update! conn :project
{:deleted-at (ct/in-future delay)}
{:deleted-at (dt/in-future delay)}
{:id project-id}
{::db/return-keys true})]

View File

@@ -11,8 +11,7 @@
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.team :as types.team]
[app.common.types.team :as tt]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -31,6 +30,7 @@
[app.setup :as-alias setup]
[app.storage :as sto]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.set :as set]))
@@ -503,7 +503,7 @@
(let [features (-> (cfeat/get-enabled-features cf/flags)
(set/difference cfeat/frontend-only-features)
(set/difference cfeat/no-team-inheritable-features))
(cfeat/check-client-features! (:features params)))
params (-> params
(assoc :profile-id profile-id)
(assoc :features features))
@@ -629,7 +629,7 @@
;; assign owner role to new profile
(db/update! conn :team-profile-rel
(get types.team/permissions-for-role :owner)
(get tt/permissions-for-role :owner)
{:team-id id :profile-id reassign-to}))
;; and finally, if all other conditions does not match and the
@@ -666,7 +666,7 @@
(let [delay (ldel/get-deletion-delay team)
team (db/update! conn :team
{:deleted-at (ct/in-future delay)}
{:deleted-at (dt/in-future delay)}
{:id id}
{::db/return-keys true})]
@@ -742,7 +742,7 @@
:team-id team-id
:role role})
(let [params (get types.team/permissions-for-role role)]
(let [params (get tt/permissions-for-role role)]
;; Only allow single owner on team
(when (= role :owner)
(db/update! conn :team-profile-rel
@@ -760,7 +760,7 @@
[:map {:title "update-team-member-role"}
[:team-id ::sm/uuid]
[:member-id ::sm/uuid]
[:role types.team/schema:role]])
[:role ::tt/role]])
(sv/defmethod ::update-team-member-role
{::doc/added "1.17"
@@ -810,7 +810,7 @@
(def ^:private schema:update-team-photo
[:map {:title "update-team-photo"}
[:team-id ::sm/uuid]
[:file media/schema:upload]])
[:file ::media/upload]])
(sv/defmethod ::update-team-photo
{::doc/added "1.17"

View File

@@ -12,7 +12,6 @@
[app.common.features :as cfeat]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.team :as types.team]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -30,6 +29,7 @@
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.util.services :as sv]
[app.util.time :as dt]
[cuerdas.core :as str]))
;; --- Mutation: Create Team Invitation
@@ -62,7 +62,7 @@
(tokens/generate (::setup/props cfg)
{:iss :profile-identity
:profile-id profile-id
:exp (ct/in-future {:days 30})}))
:exp (dt/in-future {:days 30})}))
(def ^:private schema:create-invitation
[:map {:title "params:create-invitation"}
@@ -75,7 +75,7 @@
[:map
[:id ::sm/uuid]
[:fullname :string]]]
[:role types.team/schema:role]
[:role ::types.team/role]
[:email ::sm/email]])
(def ^:private check-create-invitation-params
@@ -126,7 +126,7 @@
(teams/check-email-spam conn email true)
(let [id (uuid/next)
expire (ct/in-future "168h") ;; 7 days
expire (dt/in-future "168h") ;; 7 days
invitation (db/exec-one! conn [sql:upsert-team-invitation id
(:id team) (str/lower email)
(:id profile)
@@ -257,7 +257,7 @@
(def ^:private schema:create-team-invitations
[:map {:title "create-team-invitations"}
[:team-id ::sm/uuid]
[:role types.team/schema:role]
[:role ::types.team/role]
[:emails [::sm/set ::sm/email]]])
(def ^:private max-invitations-by-request-threshold
@@ -318,7 +318,7 @@
[:features {:optional true} ::cfeat/features]
[:id {:optional true} ::sm/uuid]
[:emails [::sm/set ::sm/email]]
[:role types.team/schema:role]])
[:role ::types.team/role]])
(sv/defmethod ::create-team-with-invitations
{::doc/added "1.17"
@@ -403,7 +403,7 @@
[:map {:title "update-team-invitation-role"}
[:team-id ::sm/uuid]
[:email ::sm/email]
[:role types.team/schema:role]])
[:role ::types.team/role]])
(sv/defmethod ::update-team-invitation-role
{::doc/added "1.17"
@@ -418,7 +418,7 @@
:code :insufficient-permissions))
(db/update! conn :team-invitation
{:role (name role) :updated-at (ct/now)}
{:role (name role) :updated-at (dt/now)}
{:team-id team-id :email-to (profile/clean-email email)})
nil))
@@ -471,7 +471,7 @@
(when-let [request (db/get* conn :team-access-request
{:team-id team-id
:requester-id profile-id})]
(when (ct/is-after? (:valid-until request) (ct/now))
(when (dt/is-after? (:valid-until request) (dt/now))
(ex/raise :type :validation
:code :request-already-sent
:hint "you have already made a request to join this team less than 24 hours ago"))))
@@ -487,8 +487,8 @@
"Create or update team access request for provided team and profile-id"
[conn team-id requester-id]
(check-existing-team-access-request conn team-id requester-id)
(let [valid-until (ct/in-future {:hours 24})
auto-join-until (ct/in-future {:days 7})
(let [valid-until (dt/in-future {:hours 24})
auto-join-until (dt/in-future {:days 7})
request-id (uuid/next)]
(db/exec-one! conn [sql:upsert-team-access-request
request-id team-id requester-id

View File

@@ -8,7 +8,6 @@
(:require
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.team :as types.team]
[app.config :as cf]
[app.db :as db]
@@ -24,7 +23,8 @@
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.tokens.spec.team-invitation :as-alias spec.team-invitation]
[app.util.services :as sv]))
[app.util.services :as sv]
[app.util.time :as dt]))
(defmulti process-token (fn [_ _ claims] (:iss claims)))
@@ -126,9 +126,9 @@
(def schema:team-invitation-claims
[:map {:title "TeamInvitationClaims"}
[:iss :keyword]
[:exp ::ct/inst]
[:exp ::dt/instant]
[:profile-id ::sm/uuid]
[:role types.team/schema:role]
[:role ::types.team/role]
[:team-id ::sm/uuid]
[:member-email ::sm/email]
[:member-id {:optional true} ::sm/uuid]])

View File

@@ -9,7 +9,6 @@
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.common.uuid :as uuid]
[app.db :as db]
@@ -20,6 +19,7 @@
[app.rpc.doc :as-alias doc]
[app.rpc.permissions :as perms]
[app.util.services :as sv]
[app.util.time :as dt]
[cuerdas.core :as str]))
(defn get-webhooks-permissions
@@ -54,7 +54,7 @@
(http/req! cfg
{:method :head
:uri (str (:uri params))
:timeout (ct/duration "3s")}
:timeout (dt/duration "3s")}
{:sync? true}))]
(if (ex/exception? response)
(if-let [hint (webhooks/interpret-exception response)]

View File

@@ -166,6 +166,9 @@
:servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
;; :description "penpot backend"
}]
:security
{:api_key []}
:paths paths
:components {:schemas @definitions}}))

View File

@@ -10,14 +10,15 @@
[app.common.exceptions :as ex]
[app.common.schema :as sm]))
(def schema:permissions
[:map {:title "Permissions"}
[:type {:gen/elements [:membership :share-link]} :keyword]
[:is-owner ::sm/boolean]
[:is-admin ::sm/boolean]
[:can-edit ::sm/boolean]
[:can-read ::sm/boolean]
[:is-logged ::sm/boolean]])
(sm/register!
^{::sm/type ::permissions}
[:map {:title "Permissions"}
[:type {:gen/elements [:membership :share-link]} :keyword]
[:is-owner ::sm/boolean]
[:is-admin ::sm/boolean]
[:can-edit ::sm/boolean]
[:can-read ::sm/boolean]
[:is-logged ::sm/boolean]])
(def valid-roles
#{:admin :owner :editor :viewer})

View File

@@ -10,9 +10,9 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]))
@@ -95,7 +95,7 @@
"- Total: ~(::total params) (INCR ~(::incr params 1))\n")]
(wrk/submit! {::db/conn conn
::wrk/task :sendmail
::wrk/delay (ct/duration "30s")
::wrk/delay (dt/duration "30s")
::wrk/max-retries 4
::wrk/priority 200
::wrk/dedupe true

View File

@@ -47,7 +47,6 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as uri]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -59,6 +58,7 @@
[app.rpc.rlimit.result :as-alias lresult]
[app.util.inet :as inet]
[app.util.services :as-alias sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.edn :as edn]
[cuerdas.core :as str]
@@ -67,7 +67,7 @@
[promesa.exec :as px]))
(def ^:private default-timeout
(ct/duration 400))
(dt/duration 400))
(def ^:private default-options
{:codec rds/string-codec
@@ -94,10 +94,6 @@
(defmulti parse-limit (fn [[_ strategy _]] strategy))
(defmulti process-limit (fn [_ _ _ o] (::strategy o)))
(defn- ->seconds
[d]
(-> d inst-ms (/ 1000) int))
(sm/register!
{:type ::rpc/rlimit
:pred #(instance? clojure.lang.Agent %)})
@@ -119,7 +115,7 @@
[:map
[::capacity ::sm/int]
[::rate ::sm/int]
[::internal ::ct/duration]
[::internal ::dt/duration]
[::params [::sm/vec :any]]]
[:map
[::nreq ::sm/int]
@@ -161,7 +157,7 @@
(assert (valid-limit-tuple? vlimit) "expected valid limit tuple")
(if-let [[_ capacity rate interval] (re-find bucket-opts-re opts)]
(let [interval (ct/duration interval)
(let [interval (dt/duration interval)
rate (parse-long rate)
capacity (parse-long capacity)]
{::name name
@@ -170,7 +166,7 @@
::rate rate
::interval interval
::opts opts
::params [(->seconds interval) rate capacity]
::params [(dt/->seconds interval) rate capacity]
::key (str "ratelimit.bucket." (d/name name))})
(ex/raise :type :validation
:code :invalid-bucket-limit-opts
@@ -180,7 +176,7 @@
[redis user-id now {:keys [::key ::params ::service ::capacity ::interval ::rate] :as limit}]
(let [script (-> bucket-rate-limit-script
(assoc ::rscript/keys [(str key "." service "." user-id)])
(assoc ::rscript/vals (conj params (->seconds now))))
(assoc ::rscript/vals (conj params (dt/->seconds now))))
result (rds/eval redis script)
allowed? (boolean (nth result 0))
remaining (nth result 1)
@@ -195,16 +191,16 @@
:remaining remaining)
(-> limit
(assoc ::lresult/allowed allowed?)
(assoc ::lresult/reset (ct/plus now reset))
(assoc ::lresult/reset (dt/plus now reset))
(assoc ::lresult/remaining remaining))))
(defmethod process-limit :window
[redis user-id now {:keys [::nreq ::unit ::key ::service] :as limit}]
(let [ts (ct/truncate now unit)
ttl (ct/diff now (ct/plus ts {unit 1}))
(let [ts (dt/truncate now unit)
ttl (dt/diff now (dt/plus ts {unit 1}))
script (-> window-rate-limit-script
(assoc ::rscript/keys [(str key "." service "." user-id "." (ct/format-inst ts))])
(assoc ::rscript/vals [nreq (->seconds ttl)]))
(assoc ::rscript/keys [(str key "." service "." user-id "." (dt/format-instant ts))])
(assoc ::rscript/vals [nreq (dt/->seconds ttl)]))
result (rds/eval redis script)
allowed? (boolean (nth result 0))
remaining (nth result 1)]
@@ -218,7 +214,7 @@
(-> limit
(assoc ::lresult/allowed allowed?)
(assoc ::lresult/remaining remaining)
(assoc ::lresult/reset (ct/plus ts {unit 1})))))
(assoc ::lresult/reset (dt/plus ts {unit 1})))))
(defn- process-limits!
[redis user-id limits now]
@@ -227,7 +223,7 @@
(d/index-by ::name ::lresult/remaining)
(uri/map->query-string))
reset (->> results
(d/index-by ::name (comp ->seconds ::lresult/reset))
(d/index-by ::name (comp dt/->seconds ::lresult/reset))
(uri/map->query-string))
rejected (d/seek (complement ::lresult/allowed) results)]
@@ -265,7 +261,7 @@
(let [redis (rds/get-or-connect redis ::rpc/rlimit default-options)
uid (get-uid params)
;; FIXME: why not clasic try/catch?
result (ex/try! (process-limits! redis uid limits (ct/now)))]
result (ex/try! (process-limits! redis uid limits (dt/now)))]
(l/trc :hint "process-limits"
:service sname
@@ -325,7 +321,7 @@
(sm/check-fn schema:config))
(def ^:private check-refresh
(sm/check-fn ::ct/duration))
(sm/check-fn ::dt/duration))
(def ^:private check-limits
(sm/check-fn schema:limits))
@@ -355,7 +351,7 @@
config)))]
(when-let [config (some->> path slurp edn/read-string check-config)]
(let [refresh (->> config meta :refresh ct/duration check-refresh)
(let [refresh (->> config meta :refresh dt/duration check-refresh)
limits (->> config compile-pass-1 compile-pass-2 check-limits)]
{::refresh refresh
@@ -414,7 +410,7 @@
(l/info :hint "initializing rlimit config reader" :path (str path))
;; Initialize the state with initial refresh value
(send-via executor state (constantly {::refresh (ct/duration "5s")}))
(send-via executor state (constantly {::refresh (dt/duration "5s")}))
;; Force a refresh
(refresh-config (assoc cfg ::path path ::state state)))

View File

@@ -11,11 +11,11 @@
[app.common.exceptions :as ex]
[app.common.json :as json]
[app.common.logging :as l]
[app.common.time :as ct]
[app.config :as cf]
[app.srepl.cli :as cli]
[app.srepl.main]
[app.util.locks :as locks]
[app.util.time :as dt]
[clojure.core :as c]
[clojure.core.server :as ccs]
[clojure.main :as cm]
@@ -77,7 +77,7 @@
(loop []
(when (try
(let [data (read-line)
tpoint (ct/tpoint)]
tpoint (dt/tpoint)]
(l/dbg :hint "received" :data (if (= data ::eof) "EOF" data))

View File

@@ -10,14 +10,13 @@
[app.auth :as auth]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.schema.generators :as sg]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.db :as db]
[app.rpc.commands.auth :as cmd.auth]
[app.rpc.commands.profile :as cmd.profile]
[app.setup :as-alias setup]
[app.tokens :as tokens]
[app.util.time :as dt]
[cuerdas.core :as str]))
(defn coercer
@@ -102,7 +101,7 @@
(fn [{:keys [::db/conn] :as system}]
(let [res (if soft
(db/update! conn :profile
{:deleted-at (ct/now)}
{:deleted-at (dt/now)}
{:email email :deleted-at nil})
(db/delete! conn :profile
{:email email}))]
@@ -174,21 +173,6 @@
:num-editors (get-customer-slots system id)
:subscription (get props :subscription)})))
(def ^:private schema:timestamp
(sm/type-schema
{:type ::timestamp
:pred ct/inst?
:type-properties
{:title "inst"
:description "The same as :app.common.time/inst but encodes to epoch"
:error/message "should be an instant"
:gen/gen (->> (sg/small-int)
(sg/fmap (fn [v] (ct/inst v))))
:decode/string ct/inst
:encode/string inst-ms
:decode/json ct/inst
:encode/json inst-ms}}))
(def ^:private schema:customer-subscription
[:map {:title "CustomerSubscription"}
[:id ::sm/text]
@@ -202,7 +186,7 @@
"canceled"
"incomplete"
"incomplete_expired"
"past_due"
"pass_due"
"paused"
"trialing"
"unpaid"]]
@@ -214,15 +198,15 @@
"year"]]
[:quantity :int]
[:description [:maybe ::sm/text]]
[:created-at schema:timestamp]
[:start-date [:maybe schema:timestamp]]
[:ended-at [:maybe schema:timestamp]]
[:trial-end [:maybe schema:timestamp]]
[:trial-start [:maybe schema:timestamp]]
[:cancel-at [:maybe schema:timestamp]]
[:canceled-at [:maybe schema:timestamp]]
[:current-period-end [:maybe schema:timestamp]]
[:current-period-start [:maybe schema:timestamp]]
[:created-at ::sm/timestamp]
[:start-date [:maybe ::sm/timestamp]]
[:ended-at [:maybe ::sm/timestamp]]
[:trial-end [:maybe ::sm/timestamp]]
[:trial-start [:maybe ::sm/timestamp]]
[:cancel-at [:maybe ::sm/timestamp]]
[:canceled-at [:maybe ::sm/timestamp]]
[:current-period-end [:maybe ::sm/timestamp]]
[:current-period-start [:maybe ::sm/timestamp]]
[:cancel-at-period-end :boolean]
[:cancellation-details

View File

@@ -12,11 +12,11 @@
[app.common.data :as d]
[app.common.files.migrations :as fmg]
[app.common.files.validate :as cfv]
[app.common.time :as ct]
[app.db :as db]
[app.main :as main]
[app.rpc.commands.files :as files]
[app.rpc.commands.files-snapshot :as fsnap]))
[app.rpc.commands.files-snapshot :as fsnap]
[app.util.time :as dt]))
(def ^:dynamic *system* nil)
@@ -165,7 +165,7 @@
(when (string? label)
(fsnap/create-file-snapshot! system file
{:label label
:deleted-at (ct/in-future {:days 30})
:deleted-at (dt/in-future {:days 30})
:created-by :admin}))
(let [file' (update file' :revn inc)]

View File

@@ -19,7 +19,6 @@
[app.common.pprint :as p]
[app.common.schema :as sm]
[app.common.spec :as us]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
@@ -39,8 +38,8 @@
[app.srepl.helpers :as h]
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.datafy :refer [datafy]]
[clojure.java.io :as io]
[clojure.pprint :refer [print-table]]
[clojure.stacktrace :as strace]
@@ -477,7 +476,7 @@
:max-jobs max-jobs
:max-items max-items)
(let [tpoint (ct/tpoint)
(let [tpoint (dt/tpoint)
factory (px/thread-factory :virtual false :prefix "penpot/file-process/")
executor (px/cached-executor :factory factory)
sjobs (ps/create :permits max-jobs)
@@ -507,7 +506,7 @@
(Thread/sleep (int pause)))
(ps/release! sjobs)
(let [elapsed (ct/format-duration (tpoint))]
(let [elapsed (dt/format-duration (tpoint))]
(l/trc :hint "process:file:end"
:tid thread-id
:file-id (str file-id)
@@ -517,7 +516,7 @@
process-file*
(fn [idx file-id]
(ps/acquire! sjobs)
(px/run! executor (partial process-file file-id idx (ct/tpoint)))
(px/run! executor (partial process-file file-id idx (dt/tpoint)))
(inc idx))
process-files
@@ -543,7 +542,7 @@
(l/dbg :hint "process:error" :cause cause))
(finally
(let [elapsed (ct/format-duration (tpoint))]
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "process:end"
:rollback rollback?
:elapsed elapsed))))))
@@ -557,7 +556,7 @@
"Mark a project for deletion"
[file-id]
(let [file-id (h/parse-uuid file-id)
tnow (ct/now)]
tnow (dt/now)]
(audit/insert! main/system
{::audit/name "delete-file"
@@ -619,7 +618,7 @@
::audit/props file
::audit/context {:triggered-by "srepl"
:cause "explicit call to restore-file!"}
::audit/tracked-at (ct/now)})
::audit/tracked-at (dt/now)})
(restore-file* system file-id))))))
@@ -627,7 +626,7 @@
"Mark a project for deletion"
[project-id]
(let [project-id (h/parse-uuid project-id)
tnow (ct/now)]
tnow (dt/now)]
(audit/insert! main/system
{::audit/name "delete-project"
@@ -674,7 +673,7 @@
::audit/props project
::audit/context {:triggered-by "srepl"
:cause "explicit call to restore-team!"}
::audit/tracked-at (ct/now)})
::audit/tracked-at (dt/now)})
(restore-project* system project-id))))))
@@ -682,7 +681,7 @@
"Mark a team for deletion"
[team-id]
(let [team-id (h/parse-uuid team-id)
tnow (ct/now)]
tnow (dt/now)]
(audit/insert! main/system
{::audit/name "delete-team"
@@ -734,7 +733,7 @@
::audit/props team
::audit/context {:triggered-by "srepl"
:cause "explicit call to restore-team!"}
::audit/tracked-at (ct/now)})
::audit/tracked-at (dt/now)})
(restore-team* system team-id))))))
@@ -742,7 +741,7 @@
"Mark a profile for deletion."
[profile-id]
(let [profile-id (h/parse-uuid profile-id)
tnow (ct/now)]
tnow (dt/now)]
(audit/insert! main/system
{::audit/name "delete-profile"
@@ -776,7 +775,7 @@
::audit/props (audit/profile->props profile)
::audit/context {:triggered-by "srepl"
:cause "explicit call to restore-profile!"}
::audit/tracked-at (ct/now)})
::audit/tracked-at (dt/now)})
(db/update! system :profile
{:deleted-at nil}
@@ -822,7 +821,7 @@
{:deleted deleted :total total})))]
(let [path (fs/path path)
deleted-at (ct/minus (ct/now) (cf/get-deletion-delay))]
deleted-at (dt/minus (dt/now) (cf/get-deletion-delay))]
(when-not (fs/exists? path)
(throw (ex-info "path does not exists" {:path path})))
@@ -906,7 +905,7 @@
(db/tx-run! main/system
(fn [{:keys [::db/conn] :as cfg}]
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
(let [team (-> (assoc cfg ::bfc/timestamp (ct/now))
(let [team (-> (assoc cfg ::bfc/timestamp (dt/now))
(mgmt/duplicate-team :team-id team-id :name name))
rels (db/query conn :team-profile-rel {:team-id team-id})]

View File

@@ -12,13 +12,13 @@
[app.common.data.macros :as dm]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.storage.fs :as sfs]
[app.storage.impl :as impl]
[app.storage.s3 :as ss3]
[app.util.time :as dt]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[integrant.core :as ig])
@@ -122,7 +122,7 @@
(dissoc :id))
touched-at (if touch
(or touched-at (ct/now))
(or touched-at (dt/now))
touched-at)
;; NOTE: for now we don't reuse the deleted objects, but in
@@ -224,7 +224,7 @@
(assert (valid-storage? storage))
(let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id)]
(-> (db/update! connectable :storage-object
{:touched-at (ct/now)}
{:touched-at (dt/now)}
{:id id})
(db/get-update-count)
(pos?))))
@@ -235,7 +235,7 @@
[storage object]
(assert (valid-storage? storage))
(when (or (nil? (:expired-at object))
(ct/is-after? (:expired-at object) (ct/now)))
(dt/is-after? (:expired-at object) (dt/now)))
(-> (impl/resolve-backend storage (:backend object))
(impl/get-object-data object))))
@@ -244,7 +244,7 @@
[storage object]
(assert (valid-storage? storage))
(when (or (nil? (:expired-at object))
(ct/is-after? (:expired-at object) (ct/now)))
(dt/is-after? (:expired-at object) (dt/now)))
(-> (impl/resolve-backend storage (:backend object))
(impl/get-object-bytes object))))
@@ -254,7 +254,7 @@
([storage object options]
(assert (valid-storage? storage))
(when (or (nil? (:expired-at object))
(ct/is-after? (:expired-at object) (ct/now)))
(dt/is-after? (:expired-at object) (dt/now)))
(-> (impl/resolve-backend storage (:backend object))
(impl/get-object-url object options)))))
@@ -266,7 +266,7 @@
(let [backend (impl/resolve-backend storage (:backend object))]
(when (and (= :fs (::type backend))
(or (nil? (:expired-at object))
(ct/is-after? (:expired-at object) (ct/now))))
(dt/is-after? (:expired-at object) (dt/now))))
(-> (impl/get-object-url backend object nil) file-url->path))))
(defn del-object!
@@ -274,7 +274,7 @@
(assert (valid-storage? storage))
(let [id (if (impl/object? object-or-id) (:id object-or-id) object-or-id)
res (db/update! connectable :storage-object
{:deleted-at (ct/now)}
{:deleted-at (dt/now)}
{:id id})]
(pos? (db/get-update-count res))))

View File

@@ -15,10 +15,10 @@
(:require
[app.common.data :as d]
[app.common.logging :as l]
[app.common.time :as ct]
[app.db :as db]
[app.storage :as sto]
[app.storage.impl :as impl]
[app.util.time :as dt]
[integrant.core :as ig]))
(def ^:private sql:lock-sobjects
@@ -106,18 +106,18 @@
(defmethod ig/expand-key ::handler
[k v]
{k (assoc v ::min-age (ct/duration {:hours 2}))})
{k (assoc v ::min-age (dt/duration {:hours 2}))})
(defmethod ig/init-key ::handler
[_ {:keys [::min-age] :as cfg}]
(fn [{:keys [props] :as task}]
(let [min-age (ct/duration (or (:min-age props) min-age))]
(let [min-age (dt/duration (or (:min-age props) min-age))]
(db/tx-run! cfg (fn [cfg]
(let [cfg (assoc cfg ::min-age min-age)
total (clean-deleted! cfg)]
(l/inf :hint "task finished"
:min-age (ct/format-duration min-age)
:min-age (dt/format-duration min-age)
:total total)
{:deleted total}))))))

View File

@@ -212,8 +212,8 @@
deleted 0]
(if-let [chunk (get-chunk pool)]
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
(recur (long (+ freezed nfo))
(long (+ deleted ndo))))
(recur (+ freezed nfo)
(+ deleted ndo)))
(do
(l/inf :hint "task finished"
:to-freeze freezed

View File

@@ -12,11 +12,11 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uri :as u]
[app.storage :as-alias sto]
[app.storage.impl :as impl]
[app.storage.tmp :as tmp]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[clojure.java.io :as io]
[datoteka.fs :as fs]
@@ -69,7 +69,7 @@
20000)
(def default-timeout
(ct/duration {:seconds 30}))
(dt/duration {:seconds 30}))
(declare put-object)
(declare get-object-bytes)
@@ -338,11 +338,11 @@
(p/fmap #(.asByteArray ^ResponseBytes %)))))
(def default-max-age
(ct/duration {:minutes 10}))
(dt/duration {:minutes 10}))
(defn- get-object-url
[{:keys [::presigner ::bucket ::prefix]} {:keys [id]} {:keys [max-age] :or {max-age default-max-age}}]
(assert (ct/duration? max-age) "expected valid duration instance")
(assert (dt/duration? max-age) "expected valid duration instance")
(let [gor (.. (GetObjectRequest/builder)
(bucket bucket)

View File

@@ -12,8 +12,8 @@
(:require
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.util.time :as dt]
[app.worker :as wrk]
[datoteka.fs :as fs]
[datoteka.io :as io]
@@ -38,7 +38,7 @@
(defmethod ig/expand-key ::cleaner
[k v]
{k (assoc v ::min-age (ct/duration "60m"))})
{k (assoc v ::min-age (dt/duration "60m"))})
(defmethod ig/init-key ::cleaner
[_ cfg]
@@ -52,13 +52,13 @@
(defn- io-loop
[{:keys [::min-age] :as cfg}]
(l/inf :hint "started tmp cleaner" :default-min-age (ct/format-duration min-age))
(l/inf :hint "started tmp cleaner" :default-min-age (dt/format-duration min-age))
(try
(loop []
(when-let [[path min-age'] (sp/take! queue)]
(let [min-age (or min-age' min-age)]
(l/dbg :hint "schedule tempfile deletion" :path path
:expires-at (ct/plus (ct/now) min-age))
:expires-at (dt/plus (dt/now) min-age))
(px/schedule! (inst-ms min-age) (partial remove-temp-file cfg path))
(recur))))
(catch InterruptedException _
@@ -87,7 +87,7 @@
path (fs/join default-tmp-dir (str prefix (uuid/next) suffix))
path (Files/createFile path attrs)]
(fs/delete-on-exit! path)
(sp/offer! queue [path (some-> min-age ct/duration)])
(sp/offer! queue [path (some-> min-age dt/duration)])
path))
(defn tempfile-from

View File

@@ -8,10 +8,10 @@
"A generic task for object deletion cascade handling"
(:require
[app.common.logging :as l]
[app.common.time :as ct]
[app.db :as db]
[app.rpc.commands.files :as files]
[app.rpc.commands.profile :as profile]
[app.util.time :as dt]
[integrant.core :as ig]))
(def ^:dynamic *team-deletion* false)
@@ -23,7 +23,7 @@
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
(when-let [file (db/get* conn :file {:id id} {::db/remove-deleted false})]
(l/trc :hint "marking for deletion" :rel "file" :id (str id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(db/update! conn :file
{:deleted-at deleted-at}
@@ -62,7 +62,7 @@
(defmethod delete-object :project
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
(l/trc :hint "marking for deletion" :rel "project" :id (str id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(db/update! conn :project
{:deleted-at deleted-at}
@@ -79,7 +79,7 @@
(defmethod delete-object :team
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
(l/trc :hint "marking for deletion" :rel "team" :id (str id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(db/update! conn :team
{:deleted-at deleted-at}
{:id id}
@@ -101,7 +101,7 @@
(defmethod delete-object :profile
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
(l/trc :hint "marking for deletion" :rel "profile" :id (str id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(db/update! conn :profile
{:deleted-at deleted-at}

View File

@@ -16,7 +16,6 @@
[app.common.files.validate :as cfv]
[app.common.logging :as l]
[app.common.thumbnails :as thc]
[app.common.time :as ct]
[app.common.types.components-list :as ctkl]
[app.common.types.file :as ctf]
[app.common.types.shape-tree :as ctt]
@@ -24,6 +23,7 @@
[app.db :as db]
[app.features.fdata :as feat.fdata]
[app.storage :as sto]
[app.util.time :as dt]
[app.worker :as wrk]
[integrant.core :as ig]))
@@ -282,7 +282,7 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [min-age (ct/duration (or (:min-age props)
(let [min-age (dt/duration (or (:min-age props)
(cf/get-deletion-delay)))
file-id (get props :file-id)
cfg (-> cfg

View File

@@ -8,9 +8,9 @@
"A maintenance task that is responsible of properly scheduling the
file-gc task for all files that matches the eligibility threshold."
(:require
[app.common.time :as ct]
[app.config :as cf]
[app.db :as db]
[app.util.time :as dt]
[app.worker :as wrk]
[integrant.core :as ig]))
@@ -53,7 +53,7 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [min-age (ct/duration (or (:min-age props) (::min-age cfg)))]
(let [min-age (dt/duration (or (:min-age props) (::min-age cfg)))]
(-> cfg
(assoc ::db/rollback (:rollback? props))
(assoc ::min-age min-age)

View File

@@ -9,9 +9,9 @@
of deleted or unreachable objects."
(:require
[app.common.logging :as l]
[app.common.time :as ct]
[app.db :as db]
[app.storage :as sto]
[app.util.time :as dt]
[integrant.core :as ig]))
(def ^:private sql:get-profiles
@@ -53,7 +53,7 @@
(l/trc :hint "permanently delete"
:rel "team"
:id (str id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; Mark as deleted the storage object
(some->> photo-id (sto/touch-object! storage))
@@ -82,7 +82,7 @@
:rel "team-font-variant"
:id (str id)
:team-id (str team-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; Mark as deleted the all related storage objects
(some->> (:woff1-file-id font) (sto/touch-object! storage))
@@ -114,7 +114,7 @@
:rel "project"
:id (str id)
:team-id (str team-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; And finally, permanently delete the project.
(db/delete! conn :project {:id id})
@@ -140,7 +140,7 @@
:rel "file"
:id (str id)
:project-id (str project-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(when (= "objects-storage" (:data-backend file))
(sto/touch-object! storage (:data-ref-id file)))
@@ -169,7 +169,7 @@
:rel "file-thumbnail"
:file-id (str file-id)
:revn revn
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; Mark as deleted the storage object
(some->> media-id (sto/touch-object! storage))
@@ -198,7 +198,7 @@
:rel "file-tagged-object-thumbnail"
:file-id (str file-id)
:object-id object-id
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; Mark as deleted the storage object
(some->> media-id (sto/touch-object! storage))
@@ -227,7 +227,7 @@
:rel "file-data-fragment"
:id (str id)
:file-id (str file-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(some->> data-ref-id (sto/touch-object! storage))
(db/delete! conn :file-data-fragment {:file-id file-id :id id})
@@ -253,7 +253,7 @@
:rel "file-media-object"
:id (str id)
:file-id (str file-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
;; Mark as deleted the all related storage objects
(some->> (:media-id fmo) (sto/touch-object! storage))
@@ -282,7 +282,7 @@
:rel "file-change"
:id (str id)
:file-id (str file-id)
:deleted-at (ct/format-inst deleted-at))
:deleted-at (dt/format-instant deleted-at))
(when (= "objects-storage" (:data-backend xlog))
(sto/touch-object! storage (:data-ref-id xlog)))
@@ -313,7 +313,7 @@
(db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
(proc-fn cfg)))]
(if (pos? result)
(recur (long (+ total result)))
(recur (+ total result))
total))))
(defmethod ig/assert-key ::handler
@@ -328,14 +328,14 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(let [threshold (ct/duration (get props :deletion-threshold 0))
(let [threshold (dt/duration (get props :deletion-threshold 0))
cfg (assoc cfg ::deletion-threshold (db/interval threshold))]
(loop [procs (map deref deletion-proc-vars)
total 0]
(if-let [proc-fn (first procs)]
(let [result (execute-proc! cfg proc-fn)]
(recur (rest procs)
(long (+ total result))))
(+ total result)))
(do
(l/inf :hint "task finished" :deleted total)
{:processed total}))))))

View File

@@ -10,8 +10,8 @@
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.common.transit :as t]
[app.util.time :as dt]
[buddy.sign.jwe :as jwe]))
(defn generate
@@ -22,7 +22,7 @@
(bytes? tokens-key))
(let [payload (-> claims
(assoc :iat (ct/now))
(assoc :iat (dt/now))
(d/without-nils)
(t/encode))]
(jwe/encrypt payload tokens-key {:alg :a256kw :enc :a256gcm})))
@@ -35,8 +35,8 @@
(defn verify
[sprops {:keys [token] :as params}]
(let [claims (decode sprops token)]
(when (and (ct/inst? (:exp claims))
(ct/is-before? (:exp claims) (ct/now)))
(when (and (dt/instant? (:exp claims))
(dt/is-before? (:exp claims) (dt/now)))
(ex/raise :type :validation
:code :invalid-token
:reason :token-expired

View File

@@ -9,7 +9,7 @@
(:refer-clojure :exclude [get])
(:require
[app.common.schema :as sm]
[app.common.time :as ct]
[app.util.time :as dt]
[promesa.exec :as px])
(:import
com.github.benmanes.caffeine.cache.AsyncCache
@@ -51,7 +51,7 @@
(let [cache (as-> (Caffeine/newBuilder) builder
(if (fn? on-remove) (.removalListener builder (create-listener on-remove)) builder)
(if executor (.executor builder ^Executor (px/resolve-executor executor)) builder)
(if keepalive (.expireAfterAccess builder ^Duration (ct/duration keepalive)) builder)
(if keepalive (.expireAfterAccess builder ^Duration (dt/duration keepalive)) builder)
(if (int? max-size) (.maximumSize builder (long max-size)) builder)
(.recordStats builder)
(.buildAsync builder))

View File

@@ -1,138 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.util.cron
(:require
[app.common.exceptions :as ex])
(:import
java.time.Instant
java.util.Date
org.apache.logging.log4j.core.util.CronExpression))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Cron Expression
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Cron expressions are comprised of 6 required fields and one
;; optional field separated by white space. The fields respectively
;; are described as follows:
;;
;; Field Name Allowed Values Allowed Special Characters
;; Seconds 0-59 , - * /
;; Minutes 0-59 , - * /
;; Hours 0-23 , - * /
;; Day-of-month 1-31 , - * ? / L W
;; Month 0-11 or JAN-DEC , - * /
;; Day-of-Week 1-7 or SUN-SAT , - * ? / L #
;; Year (Optional) empty, 1970-2199 , - * /
;;
;; The '*' character is used to specify all values. For example, "*"
;; in the minute field means "every minute".
;;
;; The '?' character is allowed for the day-of-month and day-of-week
;; fields. It is used to specify 'no specific value'. This is useful
;; when you need to specify something in one of the two fields, but
;; not the other.
;;
;; The '-' character is used to specify ranges For example "10-12" in
;; the hour field means "the hours 10, 11 and 12".
;;
;; The ',' character is used to specify additional values. For
;; example "MON,WED,FRI" in the day-of-week field means "the days
;; Monday, Wednesday, and Friday".
;;
;; The '/' character is used to specify increments. For example "0/15"
;; in the seconds field means "the seconds 0, 15, 30, and
;; 45". And "5/15" in the seconds field means "the seconds 5, 20, 35,
;; and 50". Specifying '*' before the '/' is equivalent to specifying
;; 0 is the value to start with. Essentially, for each field in the
;; expression, there is a set of numbers that can be turned on or
;; off. For seconds and minutes, the numbers range from 0 to 59. For
;; hours 0 to 23, for days of the month 0 to 31, and for months 0 to
;; 11 (JAN to DEC). The "/" character simply helps you turn on
;; every "nth" value in the given set. Thus "7/6" in the month field
;; only turns on month "7", it does NOT mean every 6th month, please
;; note that subtlety.
;;
;; The 'L' character is allowed for the day-of-month and day-of-week
;; fields. This character is short-hand for "last", but it has
;; different meaning in each of the two fields. For example, the
;; value "L" in the day-of-month field means "the last day of the
;; month" - day 31 for January, day 28 for February on non-leap
;; years. If used in the day-of-week field by itself, it simply
;; means "7" or "SAT". But if used in the day-of-week field after
;; another value, it means "the last xxx day of the month" - for
;; example "6L" means "the last friday of the month". You can also
;; specify an offset from the last day of the month, such as "L-3"
;; which would mean the third-to-last day of the calendar month. When
;; using the 'L' option, it is important not to specify lists, or
;; ranges of values, as you'll get confusing/unexpected results.
;;
;; The 'W' character is allowed for the day-of-month field. This
;; character is used to specify the weekday (Monday-Friday) nearest
;; the given day. As an example, if you were to specify "15W" as the
;; value for the day-of-month field, the meaning is: "the nearest
;; weekday to the 15th of the month". So if the 15th is a Saturday,
;; the trigger will fire on Friday the 14th. If the 15th is a Sunday,
;; the trigger will fire on Monday the 16th. If the 15th is a Tuesday,
;; then it will fire on Tuesday the 15th. However if you specify "1W"
;; as the value for day-of-month, and the 1st is a Saturday, the
;; trigger will fire on Monday the 3rd, as it will not 'jump' over the
;; boundary of a month's days. The 'W' character can only be specified
;; when the day-of-month is a single day, not a range or list of days.
;;
;; The 'L' and 'W' characters can also be combined for the
;; day-of-month expression to yield 'LW', which translates to "last
;; weekday of the month".
;;
;; The '#' character is allowed for the day-of-week field. This
;; character is used to specify "the nth" XXX day of the month. For
;; example, the value of "6#3" in the day-of-week field means the
;; third Friday of the month (day 6 = Friday and "#3" = the 3rd one in
;; the month). Other examples: "2#1" = the first Monday of the month
;; and "4#5" = the fifth Wednesday of the month. Note that if you
;; specify "#5" and there is not 5 of the given day-of-week in the
;; month, then no firing will occur that month. If the '#' character
;; is used, there can only be one expression in the day-of-week
;; field ("3#1,6#3" is not valid, since there are two expressions).
;;
;; The legal characters and the names of months and days of the week
;; are not case sensitive.
(defn cron
"Creates an instance of CronExpression from string."
[s]
(try
(CronExpression. s)
(catch java.text.ParseException e
(ex/raise :type :parse
:code :invalid-cron-expression
:cause e
:context {:expr s}))))
(defn cron-expr?
[v]
(instance? CronExpression v))
(defn next-valid-instant-from
[^CronExpression cron ^Instant now]
(assert (cron-expr? cron))
(.toInstant (.getNextValidTimeAfter cron (Date/from now))))
(defn get-next
[cron tnow]
(let [nt (next-valid-instant-from cron tnow)]
(cons nt (lazy-seq (get-next cron nt)))))
(defmethod print-method CronExpression
[o w]
(print-dup o w))
(defmethod print-dup CronExpression
[mv ^java.io.Writer writer]
;; Do not delete this comment
;; (print-ctor o (fn [o w] (print-dup (.toString ^CronExpression o) w)) w)
(.write writer (str "#penpot/cron \"" (.toString ^CronExpression mv) "\"")))

View File

@@ -37,9 +37,9 @@
(:require
[app.common.fressian :as fres]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.util.time :as dt]
[clojure.core :as c]
[clojure.data.json :as json])
(:import
@@ -61,10 +61,8 @@
(declare create)
(defn create-tracked
[& {:keys [inherit]}]
(if inherit
(atom (if *tracked* @*tracked* {}))
(atom {})))
[]
(atom {}))
(defprotocol IPointerMap
(get-id [_])
@@ -104,7 +102,7 @@
(clone [this]
(when-not loaded? (load! this))
(let [mdata (assoc mdata :created-at (ct/now))
(let [mdata (assoc mdata :created-at (dt/now))
id (uuid/next)
pmap (PointerMap. id
mdata
@@ -179,7 +177,7 @@
(let [odata' (assoc odata key val)]
(if (identical? odata odata')
this
(let [mdata (assoc mdata :created-at (ct/now))
(let [mdata (assoc mdata :created-at (dt/now))
id (if modified? id (uuid/next))
pmap (PointerMap. id
mdata
@@ -197,7 +195,7 @@
(let [odata' (dissoc odata key)]
(if (identical? odata odata')
this
(let [mdata (assoc mdata :created-at (ct/now))
(let [mdata (assoc mdata :created-at (dt/now))
id (if modified? id (uuid/next))
pmap (PointerMap. id
mdata
@@ -220,7 +218,7 @@
(defn create
([]
(let [id (uuid/next)
mdata (assoc *metadata* :created-at (ct/now))
mdata (assoc *metadata* :created-at (dt/now))
pmap (PointerMap. id mdata {} true true)]
(some-> *tracked* (swap! assoc id pmap))
pmap))
@@ -239,7 +237,7 @@
(do
(some-> *tracked* (swap! assoc (get-id data) data))
data)
(let [mdata (assoc (meta data) :created-at (ct/now))
(let [mdata (assoc (meta data) :created-at (dt/now))
id (uuid/next)
pmap (PointerMap. id
mdata

View File

@@ -0,0 +1,399 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.util.time
(:require
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.common.schema.openapi :as-alias oapi]
[app.common.time :as common-time]
[clojure.spec.alpha :as s]
[clojure.test.check.generators :as tgen]
[cuerdas.core :as str]
[fipp.ednize :as fez])
(:import
java.nio.file.attribute.FileTime
java.time.Duration
java.time.Instant
java.time.OffsetDateTime
java.time.ZoneId
java.time.ZonedDateTime
java.time.format.DateTimeFormatter
java.time.temporal.ChronoUnit
java.time.temporal.Temporal
java.time.temporal.TemporalAmount
java.time.temporal.TemporalUnit
java.util.Date
org.apache.logging.log4j.core.util.CronExpression))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Instant & Duration
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn temporal-unit
[o]
(if (instance? TemporalUnit o)
o
(case o
:nanos ChronoUnit/NANOS
:millis ChronoUnit/MILLIS
:micros ChronoUnit/MICROS
:seconds ChronoUnit/SECONDS
:minutes ChronoUnit/MINUTES
:hours ChronoUnit/HOURS
:days ChronoUnit/DAYS)))
;; --- DURATION
(defn- obj->duration
[params]
(reduce-kv (fn [o k v]
(.plus ^Duration o ^long v ^TemporalUnit (temporal-unit k)))
(Duration/ofMillis 0)
params))
(defn duration?
[v]
(instance? Duration v))
(defn duration
[ms-or-obj]
(cond
(string? ms-or-obj)
(Duration/parse (str "PT" ms-or-obj))
(duration? ms-or-obj)
ms-or-obj
(integer? ms-or-obj)
(Duration/ofMillis ms-or-obj)
:else
(obj->duration ms-or-obj)))
(defn ->seconds
[d]
(-> d inst-ms (/ 1000) int))
(defn diff
[t1 t2]
(Duration/between t1 t2))
(defn truncate
[o unit]
(let [unit (temporal-unit unit)]
(cond
(instance? Instant o)
(.truncatedTo ^Instant o ^TemporalUnit unit)
(instance? Duration o)
(.truncatedTo ^Duration o ^TemporalUnit unit)
:else
(throw (IllegalArgumentException. "only instant and duration allowed")))))
(s/def ::duration
(s/conformer
(fn [v]
(cond
(duration? v) v
(string? v)
(try
(duration v)
(catch java.time.format.DateTimeParseException _e
::s/invalid))
:else
::s/invalid))
(fn [v]
(subs (str v) 2))))
(extend-protocol clojure.core/Inst
java.time.Duration
(inst-ms* [v] (.toMillis ^Duration v))
OffsetDateTime
(inst-ms* [v] (.toEpochMilli (.toInstant ^OffsetDateTime v)))
FileTime
(inst-ms* [v] (.toMillis ^FileTime v)))
(defmethod print-method Duration
[mv ^java.io.Writer writer]
(.write writer (str "#app/duration \"" (str/lower (subs (str mv) 2)) "\"")))
(defmethod print-dup Duration [o w]
(print-method o w))
(extend-protocol fez/IEdn
Duration
(-edn [o]
(tagged-literal 'app/duration (str o))))
(defn format-duration
[o]
(str/lower (subs (str o) 2)))
;; --- INSTANT
(defn instant?
[v]
(instance? Instant v))
(defn instant
([s]
(cond
(instant? s) s
(int? s) (Instant/ofEpochMilli s)
:else (Instant/parse s)))
([s fmt]
(case fmt
:rfc1123 (Instant/from (.parse DateTimeFormatter/RFC_1123_DATE_TIME ^String s))
:iso (Instant/from (.parse DateTimeFormatter/ISO_INSTANT ^String s))
:iso8601 (Instant/from (.parse DateTimeFormatter/ISO_INSTANT ^String s)))))
(defn is-after?
"Analgous to: da > db"
[da db]
(.isAfter ^Instant da ^Instant db))
(defn is-before?
[da db]
(.isBefore ^Instant da ^Instant db))
(defn plus
[d ta]
(let [^TemporalAmount ta (duration ta)]
(cond
(instance? Duration d)
(.plus ^Duration d ta)
(instance? Temporal d)
(.plus ^Temporal d ta)
:else
(throw (UnsupportedOperationException. "unsupported type")))))
(defn minus
[d ta]
(let [^TemporalAmount ta (duration ta)]
(cond
(instance? Duration d)
(.minus ^Duration d ta)
(instance? Temporal d)
(.minus ^Temporal d ta)
:else
(throw (UnsupportedOperationException. "unsupported type")))))
(dm/export common-time/now)
(defn in-future
[v]
(plus (now) v))
(defn in-past
[v]
(minus (now) v))
(defn instant->zoned-date-time
[v]
(ZonedDateTime/ofInstant v (ZoneId/of "UTC")))
(defn format-instant
([v] (.format DateTimeFormatter/ISO_INSTANT ^Instant v))
([v fmt]
(case fmt
:iso
(.format DateTimeFormatter/ISO_INSTANT ^Instant v)
:iso-local-time
(.format DateTimeFormatter/ISO_LOCAL_TIME
^ZonedDateTime (instant->zoned-date-time v))
:rfc1123
(.format DateTimeFormatter/RFC_1123_DATE_TIME
^ZonedDateTime (instant->zoned-date-time v)))))
(defmethod print-method Instant
[mv ^java.io.Writer writer]
(.write writer (str "#app/instant \"" (format-instant mv) "\"")))
(defmethod print-dup Instant [o w]
(print-method o w))
(extend-protocol fez/IEdn
Instant
(-edn [o] (tagged-literal 'app/instant (format-instant o))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Cron Expression
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Cron expressions are comprised of 6 required fields and one
;; optional field separated by white space. The fields respectively
;; are described as follows:
;;
;; Field Name Allowed Values Allowed Special Characters
;; Seconds 0-59 , - * /
;; Minutes 0-59 , - * /
;; Hours 0-23 , - * /
;; Day-of-month 1-31 , - * ? / L W
;; Month 0-11 or JAN-DEC , - * /
;; Day-of-Week 1-7 or SUN-SAT , - * ? / L #
;; Year (Optional) empty, 1970-2199 , - * /
;;
;; The '*' character is used to specify all values. For example, "*"
;; in the minute field means "every minute".
;;
;; The '?' character is allowed for the day-of-month and day-of-week
;; fields. It is used to specify 'no specific value'. This is useful
;; when you need to specify something in one of the two fields, but
;; not the other.
;;
;; The '-' character is used to specify ranges For example "10-12" in
;; the hour field means "the hours 10, 11 and 12".
;;
;; The ',' character is used to specify additional values. For
;; example "MON,WED,FRI" in the day-of-week field means "the days
;; Monday, Wednesday, and Friday".
;;
;; The '/' character is used to specify increments. For example "0/15"
;; in the seconds field means "the seconds 0, 15, 30, and
;; 45". And "5/15" in the seconds field means "the seconds 5, 20, 35,
;; and 50". Specifying '*' before the '/' is equivalent to specifying
;; 0 is the value to start with. Essentially, for each field in the
;; expression, there is a set of numbers that can be turned on or
;; off. For seconds and minutes, the numbers range from 0 to 59. For
;; hours 0 to 23, for days of the month 0 to 31, and for months 0 to
;; 11 (JAN to DEC). The "/" character simply helps you turn on
;; every "nth" value in the given set. Thus "7/6" in the month field
;; only turns on month "7", it does NOT mean every 6th month, please
;; note that subtlety.
;;
;; The 'L' character is allowed for the day-of-month and day-of-week
;; fields. This character is short-hand for "last", but it has
;; different meaning in each of the two fields. For example, the
;; value "L" in the day-of-month field means "the last day of the
;; month" - day 31 for January, day 28 for February on non-leap
;; years. If used in the day-of-week field by itself, it simply
;; means "7" or "SAT". But if used in the day-of-week field after
;; another value, it means "the last xxx day of the month" - for
;; example "6L" means "the last friday of the month". You can also
;; specify an offset from the last day of the month, such as "L-3"
;; which would mean the third-to-last day of the calendar month. When
;; using the 'L' option, it is important not to specify lists, or
;; ranges of values, as you'll get confusing/unexpected results.
;;
;; The 'W' character is allowed for the day-of-month field. This
;; character is used to specify the weekday (Monday-Friday) nearest
;; the given day. As an example, if you were to specify "15W" as the
;; value for the day-of-month field, the meaning is: "the nearest
;; weekday to the 15th of the month". So if the 15th is a Saturday,
;; the trigger will fire on Friday the 14th. If the 15th is a Sunday,
;; the trigger will fire on Monday the 16th. If the 15th is a Tuesday,
;; then it will fire on Tuesday the 15th. However if you specify "1W"
;; as the value for day-of-month, and the 1st is a Saturday, the
;; trigger will fire on Monday the 3rd, as it will not 'jump' over the
;; boundary of a month's days. The 'W' character can only be specified
;; when the day-of-month is a single day, not a range or list of days.
;;
;; The 'L' and 'W' characters can also be combined for the
;; day-of-month expression to yield 'LW', which translates to "last
;; weekday of the month".
;;
;; The '#' character is allowed for the day-of-week field. This
;; character is used to specify "the nth" XXX day of the month. For
;; example, the value of "6#3" in the day-of-week field means the
;; third Friday of the month (day 6 = Friday and "#3" = the 3rd one in
;; the month). Other examples: "2#1" = the first Monday of the month
;; and "4#5" = the fifth Wednesday of the month. Note that if you
;; specify "#5" and there is not 5 of the given day-of-week in the
;; month, then no firing will occur that month. If the '#' character
;; is used, there can only be one expression in the day-of-week
;; field ("3#1,6#3" is not valid, since there are two expressions).
;;
;; The legal characters and the names of months and days of the week
;; are not case sensitive.
(defn cron
"Creates an instance of CronExpression from string."
[s]
(try
(CronExpression. s)
(catch java.text.ParseException e
(ex/raise :type :parse
:code :invalid-cron-expression
:cause e
:context {:expr s}))))
(defn cron?
[v]
(instance? CronExpression v))
(defn next-valid-instant-from
[^CronExpression cron ^Instant now]
(s/assert cron? cron)
(.toInstant (.getNextValidTimeAfter cron (Date/from now))))
(defn get-next
[cron tnow]
(let [nt (next-valid-instant-from cron tnow)]
(cons nt (lazy-seq (get-next cron nt)))))
(defmethod print-method CronExpression
[mv ^java.io.Writer writer]
(.write writer (str "#app/cron \"" (.toString ^CronExpression mv) "\"")))
(defmethod print-dup CronExpression
[o w]
(print-ctor o (fn [o w] (print-dup (.toString ^CronExpression o) w)) w))
(extend-protocol fez/IEdn
CronExpression
(-edn [o] (pr-str o)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Measurement Helpers
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn tpoint
"Create a measurement checkpoint for time measurement of potentially
asynchronous flow."
[]
(let [p1 (System/nanoTime)]
#(duration {:nanos (- (System/nanoTime) p1)})))
(sm/register!
{:type ::instant
:pred instant?
:type-properties
{:error/message "should be an instant"
:title "instant"
:decode/string instant
:encode/string format-instant
:decode/json instant
:encode/json format-instant
:gen/gen (tgen/fmap (fn [i] (in-past i)) tgen/pos-int)
::oapi/type "string"
::oapi/format "iso"}})
(sm/register!
{:type ::duration
:pred duration?
:type-properties
{:error/message "should be a duration"
:gen/gen (tgen/fmap duration tgen/pos-int)
:title "duration"
:decode/string duration
:encode/string format-duration
:decode/json duration
:encode/json format-duration
::oapi/type "string"
::oapi/format "duration"}})

View File

@@ -9,10 +9,10 @@
(:require
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.transit :as t]
[app.common.uuid :as uuid]
[app.util.inet :as inet]
[app.util.time :as dt]
[promesa.exec :as px]
[promesa.exec.csp :as sp]
[promesa.util :as pu]
@@ -93,7 +93,7 @@
(assoc ::id id)
(assoc ::state state)
(assoc ::beats beats)
(assoc ::created-at (ct/now))
(assoc ::created-at (dt/now))
(assoc ::input-ch input-ch)
(assoc ::heartbeat-ch hbeat-ch)
(assoc ::output-ch output-ch)
@@ -107,7 +107,7 @@
(let [options (-> options
(assoc ::channel channel)
(on-connect))
timeout (ct/duration idle-timeout)]
timeout (dt/duration idle-timeout)]
(yws/set-idle-timeout! channel timeout)
(px/submit! :vthread (partial start-io-loop! options))))
@@ -128,7 +128,7 @@
(fn on-message [_channel message]
(when (string? message)
(sp/offer! input-ch message)
(swap! state assoc ::last-activity-at (ct/now))))
(swap! state assoc ::last-activity-at (dt/now))))
:on-pong
(fn on-pong [_channel data]

View File

@@ -10,11 +10,11 @@
[app.common.data :as d]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.metrics :as mtx]
[app.util.time :as dt]
[cuerdas.core :as str]
[integrant.core :as ig]))
@@ -31,7 +31,7 @@
[f metrics tname]
(let [labels (into-array String [tname])]
(fn [params]
(let [tp (ct/tpoint)]
(let [tp (dt/tpoint)]
(try
(f params)
(finally
@@ -95,7 +95,7 @@
[::task [:or ::sm/text :keyword]]
[::label {:optional true} ::sm/text]
[::delay {:optional true}
[:or ::sm/int ::ct/duration]]
[:or ::sm/int ::dt/duration]]
[::queue {:optional true} [:or ::sm/text :keyword]]
[::priority {:optional true} ::sm/int]
[::max-retries {:optional true} ::sm/int]
@@ -111,7 +111,7 @@
(check-options! options)
(let [duration (ct/duration delay)
(let [duration (dt/duration delay)
interval (db/interval duration)
props (db/tjson params)
id (uuid/next)
@@ -129,7 +129,7 @@
:queue queue
:label label
:dedupe (boolean dedupe)
:delay (ct/format-duration duration)
:delay (dt/format-duration duration)
:replace (or deleted 0))
(db/exec-one! conn [sql:insert-new-task id task props queue

View File

@@ -10,9 +10,8 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.db :as db]
[app.util.cron :as cron]
[app.util.time :as dt]
[app.worker :as wrk]
[app.worker.runner :refer [get-error-context]]
[cuerdas.core :as str]
@@ -50,7 +49,7 @@
[cfg {:keys [id cron] :as task}]
(px/thread
{:name (str "penpot/cron-task/" id)}
(let [tpoint (ct/tpoint)]
(let [tpoint (dt/tpoint)]
(try
(db/tx-run! cfg (fn [{:keys [::db/conn]}]
(db/exec-one! conn ["SET LOCAL statement_timeout=0;"])
@@ -58,20 +57,20 @@
(when (lock-scheduled-task! conn id)
(db/update! conn :scheduled-task
{:cron-expr (str cron)
:modified-at (ct/now)}
:modified-at (dt/now)}
{:id id}
{::db/return-keys false})
(l/dbg :hint "start" :id id)
((:fn task) task)
(let [elapsed (ct/format-duration (tpoint))]
(let [elapsed (dt/format-duration (tpoint))]
(l/dbg :hint "end" :id id :elapsed elapsed)))))
(catch InterruptedException _
(let [elapsed (ct/format-duration (tpoint))]
(let [elapsed (dt/format-duration (tpoint))]
(l/debug :hint "task interrupted" :id id :elapsed elapsed)))
(catch Throwable cause
(let [elapsed (ct/format-duration (tpoint))]
(let [elapsed (dt/format-duration (tpoint))]
(binding [l/*context* (get-error-context cause task)]
(l/err :hint "unhandled exception on running task"
:id id
@@ -83,10 +82,10 @@
(defn- ms-until-valid
[cron]
(assert (cron/cron-expr? cron) "expected cron instance")
(let [now (ct/now)
next (cron/next-valid-instant-from cron now)]
(ct/diff now next)))
(assert (dt/cron? cron) "expected cron instance")
(let [now (dt/now)
next (dt/next-valid-instant-from cron now)]
(dt/diff now next)))
(defn- schedule-cron-task
[{:keys [::running] :as cfg} {:keys [cron id] :as task}]
@@ -94,8 +93,8 @@
ft (px/schedule! ts (partial execute-cron-task cfg task))]
(l/dbg :hint "schedule" :id id
:ts (ct/format-duration ts)
:at (ct/format-inst (ct/in-future ts)))
:ts (dt/format-duration ts)
:at (dt/format-instant (dt/in-future ts)))
(swap! running #(into #{ft} (filter p/pending?) %))))
@@ -105,7 +104,7 @@
[:vector
[:maybe
[:map
[:cron [:fn cron/cron-expr?]]
[:cron [:fn dt/cron?]]
[:task :keyword]
[:props {:optional true} :map]
[:id {:optional true} :keyword]]]]]

View File

@@ -10,11 +10,11 @@
[app.common.data.macros :as dm]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.transit :as t]
[app.db :as db]
[app.metrics :as mtx]
[app.redis :as rds]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[cuerdas.core :as str]
[integrant.core :as ig]
@@ -32,9 +32,9 @@
(defmethod ig/expand-key ::wrk/dispatcher
[k v]
{k (-> (d/without-nils v)
(assoc ::timeout (ct/duration "10s"))
(assoc ::timeout (dt/duration "10s"))
(assoc ::batch-size 100)
(assoc ::wait-duration (ct/duration "5s")))})
(assoc ::wait-duration (dt/duration "5s")))})
(defmethod ig/assert-key ::wrk/dispatcher
[_ cfg]

View File

@@ -10,8 +10,8 @@
[app.common.data :as d]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.metrics :as mtx]
[app.util.time :as dt]
[app.worker :as-alias wrk]
[integrant.core :as ig]
[promesa.exec :as px])
@@ -55,7 +55,7 @@
(defmethod ig/expand-key ::wrk/monitor
[k v]
{k (-> (d/without-nils v)
(assoc ::interval (ct/duration "2s")))})
(assoc ::interval (dt/duration "2s")))})
(defmethod ig/init-key ::wrk/monitor
[_ {:keys [::wrk/executor ::mtx/metrics ::interval ::wrk/name]}]

View File

@@ -12,11 +12,11 @@
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.transit :as t]
[app.db :as db]
[app.metrics :as mtx]
[app.redis :as rds]
[app.util.time :as dt]
[app.worker :as wrk]
[cuerdas.core :as str]
[integrant.core :as ig]
@@ -29,10 +29,10 @@
[:id ::sm/uuid]
[:queue :string]
[:name :string]
[:created-at ::ct/inst]
[:modified-at ::ct/inst]
[:scheduled-at {:optional true} ::ct/inst]
[:completed-at {:optional true} ::ct/inst]
[:created-at ::sm/inst]
[:modified-at ::sm/inst]
[:scheduled-at {:optional true} ::sm/inst]
[:completed-at {:optional true} ::sm/inst]
[:error {:optional true} :string]
[:max-retries :int]
[:retry-num :int]
@@ -76,10 +76,10 @@
:queue queue
:runner-id id
:retry (:retry-num task))
(let [tpoint (ct/tpoint)
(let [tpoint (dt/tpoint)
task-fn (wrk/get-task registry (:name task))
result (when task-fn (task-fn task))
elapsed (ct/format-duration (tpoint))
elapsed (dt/format-duration (tpoint))
result (if (valid-task-result? result)
result
{:status "completed"})]
@@ -105,7 +105,7 @@
(:max-retries task))
(= ::retry (:type edata)))
(cond-> {:status "retry" :error cause}
(ct/duration? (:delay edata))
(dt/duration? (:delay edata))
(assoc :delay (:delay edata))
(= ::noop (:strategy edata))
@@ -156,13 +156,13 @@
(str error))
task (-> result meta ::task)
nretry (+ (:retry-num task) inc-by)
now (ct/now)
now (dt/now)
delay (->> (iterate #(* % 2) delay) (take nretry) (last))]
(db/update! pool :task
{:error explain
:status "retry"
:modified-at now
:scheduled-at (ct/plus now delay)
:scheduled-at (dt/plus now delay)
:retry-num nretry}
{:id (:id task)})
nil))
@@ -172,14 +172,14 @@
explain (ex-message error)]
(db/update! pool :task
{:error explain
:modified-at (ct/now)
:modified-at (dt/now)
:status "failed"}
{:id (:id task)})
nil))
(handle-task-completion [result]
(let [task (-> result meta ::task)
now (ct/now)]
now (dt/now)]
(db/update! pool :task
{:completed-at now
:modified-at now
@@ -255,7 +255,7 @@
(let [cfg (-> cfg
(assoc ::rds/rconn rconn)
(assoc ::queue (str/ffmt "%:%" tenant queue))
(assoc ::timeout (ct/duration "5s")))]
(assoc ::timeout (dt/duration "5s")))]
(loop []
(when (px/interrupted?)
(throw (InterruptedException. "interrupted")))

View File

@@ -1,10 +1,3 @@
{penpot/inst app.common.time/inst
penpot/cron app.util.cron/cron
penpot/duration app.common.time/duration
penpot/path-data app.common.types.path/from-string
penpot/matrix app.common.geom.matrix/decode-matrix
penpot/point app.common.geom.point/decode-point
penpot/token-lib app.common.types.tokens-lib/parse-multi-set-dtcg-json
penpot/token-set app.common.types.tokens-lib/make-token-set
penpot/token-theme app.common.types.tokens-lib/make-token-theme
penpot/token app.common.types.tokens-lib/make-token}
{app/instant app.util.time/instant
app/cron app.util.time/cron
app/duration app.util.time/duration}

View File

@@ -20,6 +20,7 @@
[app.rpc :as-alias rpc]
[app.storage :as sto]
[app.storage.tmp :as tmp]
[app.util.time :as dt]
[backend-tests.helpers :as th]
[clojure.test :as t]
[cuerdas.core :as str]

View File

@@ -6,11 +6,11 @@
(ns backend-tests.bounce-handling-test
(:require
[app.common.time :as ct]
[app.db :as db]
[app.email :as email]
[app.http.awsns :as awsns]
[app.tokens :as tokens]
[app.util.time :as dt]
[backend-tests.helpers :as th]
[clojure.pprint :refer [pprint]]
[clojure.test :as t]
@@ -250,7 +250,7 @@
(let [profile (th/create-profile* 1)
pool (:app.db/pool th/*system*)]
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (ct/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (dt/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile)})
(th/create-complaint-for pool {:type :bounce :id (:id profile)})
@@ -268,8 +268,8 @@
:profile-complaint-threshold 2})}]
(let [profile (th/create-profile* 1)
pool (:app.db/pool th/*system*)]
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (ct/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (ct/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (dt/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile) :created-at (dt/in-past {:days 8})})
(th/create-complaint-for pool {:type :bounce :id (:id profile)})
(th/create-complaint-for pool {:type :bounce :id (:id profile)})
(th/create-complaint-for pool {:type :complaint :id (:id profile)})

View File

@@ -15,7 +15,6 @@
[app.common.pprint :as pp]
[app.common.schema :as sm]
[app.common.spec :as us]
[app.common.time :as ct]
[app.common.transit :as tr]
[app.common.uuid :as uuid]
[app.config :as cf]
@@ -34,6 +33,7 @@
[app.rpc.helpers :as rph]
[app.util.blob :as blob]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[app.worker.runner]
[clojure.java.io :as io]
@@ -263,7 +263,7 @@
(dm/with-open [conn (db/open system)]
(db/insert! conn :profile-complaint-report
{:profile-id id
:created-at (or created-at (ct/now))
:created-at (or created-at (dt/now))
:type (name type)
:content (db/tjson {})})))
@@ -273,7 +273,7 @@
(db/insert! conn :global-complaint-report
{:email email
:type (name type)
:created-at (or created-at (ct/now))
:created-at (or created-at (dt/now))
:content (db/tjson {})})))
(defn create-team-role*
@@ -305,7 +305,7 @@
([system {:keys [file-id changes session-id profile-id revn]
:or {session-id (uuid/next) revn 0}}]
(-> system
(assoc ::files.update/timestamp (ct/now))
(assoc ::files.update/timestamp (dt/now))
(db/tx-run! (fn [{:keys [::db/conn] :as system}]
(let [file (files.update/get-file conn file-id)]
(#'files.update/update-file* system
@@ -379,7 +379,7 @@
;; (app.common.pprint/pprint (:app.rpc/methods *system*))
(try-on! (method-fn (-> data
(dissoc ::type)
(assoc :app.rpc/request-at (ct/now)))))))
(assoc :app.rpc/request-at (dt/now)))))))
(defn run-task!
([name]
@@ -525,7 +525,7 @@
(defn sleep
[ms-or-duration]
(Thread/sleep (inst-ms (ct/duration ms-or-duration))))
(Thread/sleep (inst-ms (dt/duration ms-or-duration))))
(defn config-get-mock
[data]

View File

@@ -1,96 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns backend-tests.http-management-test
(:require
[app.common.data :as d]
[app.common.time :as ct]
[app.db :as db]
[app.http.access-token]
[app.http.management :as mgmt]
[app.http.session :as sess]
[app.main :as-alias main]
[app.rpc :as-alias rpc]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]
[yetti.response :as-alias yres]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(t/deftest authenticate-method
(let [profile (th/create-profile* 1)
props (get th/*system* :app.setup/props)
token (#'sess/gen-token props {:profile-id (:id profile)})
request {:params {:token token}}
response (#'mgmt/authenticate th/*system* request)]
(t/is (= 200 (::yres/status response)))
(t/is (= "authentication" (-> response ::yres/body :iss)))
(t/is (= (:id profile) (-> response ::yres/body :uid)))))
(t/deftest get-customer-method
(let [profile (th/create-profile* 1)
request {:params {:id (:id profile)}}
response (#'mgmt/get-customer th/*system* request)]
(t/is (= 200 (::yres/status response)))
(t/is (= (:id profile) (-> response ::yres/body :id)))
(t/is (= (:fullname profile) (-> response ::yres/body :name)))
(t/is (= (:email profile) (-> response ::yres/body :email)))
(t/is (= 1 (-> response ::yres/body :num-editors)))
(t/is (nil? (-> response ::yres/body :subscription)))))
(t/deftest update-customer-method
(let [profile (th/create-profile* 1)
subs {:type "unlimited"
:description nil
:id "foobar"
:customer-id (str (:id profile))
:status "past_due"
:billing-period "week"
:quantity 1
:created-at (ct/truncate (ct/now) :day)
:cancel-at-period-end true
:start-date nil
:ended-at nil
:trial-end nil
:trial-start nil
:cancel-at nil
:canceled-at nil
:current-period-end nil
:current-period-start nil
:cancellation-details
{:comment "other"
:reason "other"
:feedback "other"}}
request {:params {:id (:id profile)
:subscription subs}}
response (#'mgmt/update-customer th/*system* request)]
(t/is (= 201 (::yres/status response)))
(t/is (nil? (::yres/body response)))
(let [request {:params {:id (:id profile)}}
response (#'mgmt/get-customer th/*system* request)]
(t/is (= 200 (::yres/status response)))
(t/is (= (:id profile) (-> response ::yres/body :id)))
(t/is (= (:fullname profile) (-> response ::yres/body :name)))
(t/is (= (:email profile) (-> response ::yres/body :email)))
(t/is (= 1 (-> response ::yres/body :num-editors)))
(let [subs' (-> response ::yres/body :subscription)]
(t/is (= subs' subs))))))

Some files were not shown because too many files have changed in this diff Show More