mirror of
https://github.com/penpot/penpot.git
synced 2026-01-09 06:49:05 -05:00
Compare commits
1 Commits
pre-lazy-l
...
hiru-refac
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adc6af129c |
@@ -11,11 +11,11 @@ jobs:
|
||||
- image: cimg/redis:7.0.5
|
||||
|
||||
working_directory: ~/repo
|
||||
resource_class: medium+
|
||||
resource_class: large
|
||||
|
||||
environment:
|
||||
JAVA_OPTS: -Xmx4g -Xms100m -XX:+UseSerialGC
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
# Customize the JVM maximum heap limit
|
||||
JVM_OPTS: -Xmx4g
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
@@ -28,59 +28,50 @@ jobs:
|
||||
- v1-dependencies-
|
||||
|
||||
- run: cd .clj-kondo && cat config.edn
|
||||
- run: cat .cljfmt.edn
|
||||
- run: clj-kondo --version
|
||||
|
||||
# - run:
|
||||
# name: "fmt check [clj]"
|
||||
# command: |
|
||||
# yarn run fmt:clj:check
|
||||
- run:
|
||||
name: frontend styles prettier
|
||||
working_directory: "./frontend"
|
||||
command: |
|
||||
yarn install
|
||||
yarn run lint-scss
|
||||
|
||||
- run:
|
||||
name: common lint
|
||||
working_directory: "./common"
|
||||
command: |
|
||||
yarn install
|
||||
yarn run lint:clj
|
||||
clj-kondo --version
|
||||
clj-kondo --parallel --lint src/
|
||||
|
||||
- run:
|
||||
name: frontend lint
|
||||
working_directory: "./frontend"
|
||||
command: |
|
||||
yarn install
|
||||
yarn run lint:scss
|
||||
yarn run lint:clj
|
||||
clj-kondo --version
|
||||
clj-kondo --parallel --lint src/
|
||||
|
||||
- run:
|
||||
name: backend lint
|
||||
working_directory: "./backend"
|
||||
command: |
|
||||
yarn install
|
||||
yarn run lint:clj
|
||||
clj-kondo --version
|
||||
clj-kondo --parallel --lint src/
|
||||
|
||||
- run:
|
||||
name: exporter lint
|
||||
working_directory: "./exporter"
|
||||
command: |
|
||||
yarn install
|
||||
yarn run lint:clj
|
||||
|
||||
- run:
|
||||
name: "common tests"
|
||||
working_directory: "./common"
|
||||
name: common tests
|
||||
command: |
|
||||
yarn install
|
||||
yarn test
|
||||
clojure -X:dev:test :patterns '["common-tests.*-test"]'
|
||||
|
||||
- run:
|
||||
name: "frontend tests"
|
||||
working_directory: "./frontend"
|
||||
command: |
|
||||
yarn install
|
||||
yarn test
|
||||
environment:
|
||||
PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
|
||||
JVM_OPTS: -Xmx4g
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
|
||||
- run:
|
||||
name: "backend tests"
|
||||
name: backend test
|
||||
working_directory: "./backend"
|
||||
command: |
|
||||
clojure -X:dev:test :patterns '["backend-tests.*-test"]'
|
||||
@@ -90,6 +81,18 @@ jobs:
|
||||
PENPOT_TEST_DATABASE_USERNAME: penpot_test
|
||||
PENPOT_TEST_DATABASE_PASSWORD: penpot_test
|
||||
PENPOT_TEST_REDIS_URI: "redis://localhost/1"
|
||||
JVM_OPTS: -Xmx4g
|
||||
|
||||
- run:
|
||||
name: frontend tests
|
||||
working_directory: "./frontend"
|
||||
command: |
|
||||
yarn install
|
||||
yarn test
|
||||
|
||||
environment:
|
||||
PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
|
||||
NODE_OPTIONS: --max-old-space-size=4096
|
||||
|
||||
- save_cache:
|
||||
paths:
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
promesa.exec.csp/go-loop clojure.core/loop
|
||||
rumext.v2/defc clojure.core/defn
|
||||
rumext.v2/fnc clojure.core/fn
|
||||
promesa.util/with-open clojure.core/with-open
|
||||
app.common.data/export clojure.core/def
|
||||
app.db/with-atomic clojure.core/with-open
|
||||
app.common.data.macros/get-in clojure.core/get-in
|
||||
app.common.data.macros/with-open clojure.core/with-open
|
||||
app.common.data.macros/select-keys clojure.core/select-keys
|
||||
@@ -15,26 +15,16 @@
|
||||
:hooks
|
||||
{:analyze-call
|
||||
{app.common.data.macros/export hooks.export/export
|
||||
potok.v2.core/reify hooks.export/potok-reify
|
||||
potok.core/reify hooks.export/potok-reify
|
||||
app.util.services/defmethod hooks.export/service-defmethod
|
||||
app.common.record/defrecord hooks.export/penpot-defrecord
|
||||
app.db/with-atomic hooks.export/penpot-with-atomic
|
||||
rumext.v2/fnc hooks.export/rumext-fnc
|
||||
}}
|
||||
|
||||
:output
|
||||
{:exclude-files
|
||||
["data_readers.clj"
|
||||
"src/app/util/perf.cljs"
|
||||
"src/app/common/logging.cljc"
|
||||
"src/app/common/exceptions.cljc"
|
||||
"^(?:backend|frontend|exporter|common)/build.clj"
|
||||
"^(?:backend|frontend|exporter|common)/deps.edn"
|
||||
"^(?:backend|frontend|exporter|common)/scripts/"
|
||||
"^(?:backend|frontend|exporter|common)/dev/"
|
||||
"^(?:backend|frontend|exporter|common)/test/"]
|
||||
|
||||
:linter-name true}
|
||||
"app/util/perf.cljs"
|
||||
"app/common/logging.cljc"
|
||||
"app/common/exceptions.cljc"]}
|
||||
|
||||
:linters
|
||||
{:unsorted-required-namespaces
|
||||
@@ -70,3 +60,4 @@
|
||||
:exclude-destructured-keys-in-fn-args false
|
||||
}
|
||||
}}
|
||||
|
||||
|
||||
@@ -39,60 +39,6 @@
|
||||
other))]
|
||||
{:node result})))
|
||||
|
||||
(defn penpot-with-atomic
|
||||
[{:keys [node]}]
|
||||
(let [[params & body] (rest (:children node))]
|
||||
(if (api/vector-node? params)
|
||||
(let [[sym val opts] (:children params)]
|
||||
(when-not (and sym val)
|
||||
(throw (ex-info "No sym and val provided" {})))
|
||||
{:node (api/list-node
|
||||
(list*
|
||||
(api/token-node 'let)
|
||||
(api/vector-node [sym val])
|
||||
opts
|
||||
body))})
|
||||
|
||||
{:node (api/list-node
|
||||
(into [(api/token-node 'let)
|
||||
(api/vector-node [params params])]
|
||||
body))})))
|
||||
|
||||
(defn rumext-fnc
|
||||
[{:keys [node]}]
|
||||
(let [[cname mdata params & body] (rest (:children node))
|
||||
[params body] (if (api/vector-node? mdata)
|
||||
[mdata (cons params body)]
|
||||
[params body])]
|
||||
(let [result (api/list-node
|
||||
(into [(api/token-node 'fn)
|
||||
params]
|
||||
(cons mdata body)))]
|
||||
{:node result})))
|
||||
|
||||
|
||||
(defn penpot-defrecord
|
||||
[{:keys [:node]}]
|
||||
(let [[rnode rtype rparams & other] (:children node)
|
||||
|
||||
nodes [(api/token-node (symbol "do"))
|
||||
(api/list-node
|
||||
(into [(api/token-node (symbol (name (:value rnode)))) rtype rparams] other))
|
||||
(api/list-node
|
||||
[(api/token-node (symbol "defn"))
|
||||
(api/token-node (symbol (str "pos->" (:string-value rtype))))
|
||||
(api/vector-node
|
||||
(->> (:children rparams)
|
||||
(mapv (fn [t]
|
||||
(api/token-node (symbol (str "_" (:string-value t))))))))
|
||||
(api/token-node nil)])]
|
||||
|
||||
result (api/list-node nodes)]
|
||||
|
||||
;; (prn "=====>" (into {} rparams))
|
||||
;; (prn (api/sexpr result))
|
||||
{:node result}))
|
||||
|
||||
(defn clojure-specify
|
||||
[{:keys [:node]}]
|
||||
(let [[rnode rtype & other] (:children node)
|
||||
@@ -102,6 +48,7 @@
|
||||
other))]
|
||||
{:node result}))
|
||||
|
||||
|
||||
(defn service-defmethod
|
||||
[{:keys [:node]}]
|
||||
(let [[rnode rtype ?meta & other] (:children node)
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
{:sort-ns-references? true
|
||||
:remove-multiple-non-indenting-spaces? false
|
||||
:remove-surrounding-whitespace? true
|
||||
:remove-consecutive-blank-lines? false
|
||||
:extra-indents {rumext.v2/fnc [[:inner 0]]
|
||||
promesa.exec/thread [[:inner 0]]
|
||||
specify! [[:inner 0] [:inner 1]]}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*.{cljs,cljc,clj,js,css,scss,html,yml,yaml,json,mustache}]
|
||||
charset = utf-8
|
||||
|
||||
indent_size = 2
|
||||
indent_style = space
|
||||
|
||||
end_of_line = lf
|
||||
|
||||
insert_final_newline = true
|
||||
|
||||
trim_trailing_whitespace = true
|
||||
10
.gitignore
vendored
10
.gitignore
vendored
@@ -1,16 +1,7 @@
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
*-init.clj
|
||||
*.css.json
|
||||
*.jar
|
||||
*.orig
|
||||
*.penpot
|
||||
*.css.json
|
||||
.calva
|
||||
.clj-kondo
|
||||
.cpcache
|
||||
@@ -66,4 +57,3 @@
|
||||
/web
|
||||
clj-profiler/
|
||||
node_modules
|
||||
frontend/.storybook/preview-body.html
|
||||
|
||||
9
.vscode/settings.json
vendored
9
.vscode/settings.json
vendored
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"files.exclude": {
|
||||
"**/.clj-kondo": true,
|
||||
"**/.cpcache": true,
|
||||
"**/.lsp": true,
|
||||
"**/.shadow-cljs": true,
|
||||
"**/node_modules": true
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
enableGlobalCache: true
|
||||
|
||||
enableImmutableCache: false
|
||||
|
||||
enableImmutableInstalls: false
|
||||
|
||||
enableTelemetry: false
|
||||
|
||||
nodeLinker: node-modules
|
||||
179
CHANGES.md
179
CHANGES.md
@@ -1,197 +1,23 @@
|
||||
# CHANGELOG
|
||||
|
||||
## :rocket: Next
|
||||
## :rocket: 1.19.0
|
||||
|
||||
### :boom: Breaking changes & Deprecations
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
### :arrow_up: Deps updates
|
||||
|
||||
### :heart: Community contributions by (Thank you!)
|
||||
|
||||
## 1.20.0
|
||||
|
||||
### :boom: Breaking changes & Deprecations
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
- Select through stroke only rectangle [Taiga #5484](https://tree.taiga.io/project/penpot/issue/5484)
|
||||
- Override browser Ctrl+ and Ctrl- zoom with Penpot Zoom [Taiga #3200](https://tree.taiga.io/project/penpot/us/3200)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix pixelated thumbnails [Github
|
||||
#3681](https://github.com/penpot/penpot/issues/3681) [Github #3661](https://github.com/penpot/penpot/issues/3661)
|
||||
|
||||
### :arrow_up: Deps updates
|
||||
|
||||
### :heart: Community contributions by (Thank you!)
|
||||
|
||||
## 1.19.5
|
||||
|
||||
### :bug: New features
|
||||
|
||||
- Fix problem with alignment performance
|
||||
|
||||
## 1.19.4
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
- Improve selected colors [Taiga #5805]( https://tree.taiga.io/project/penpot/us/5805)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix problem with z-index field in non-absolute items
|
||||
|
||||
## 1.19.3
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
- Remember last color mode in colorpicker [Taiga #5508](https://tree.taiga.io/project/penpot/issue/5508)
|
||||
- Improve layers multiselection behaviour [Github #5741](https://github.com/penpot/penpot/issues/5741)
|
||||
- Remember last active team across logouts / sessions [Github #3325](https://github.com/penpot/penpot/issues/3325)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- List view is discarded on tab change on Workspace Assets Sidebar tab [Github #3547](https://github.com/penpot/penpot/issues/3547)
|
||||
- Fix message popup remains open when exiting workspace with browser back button [Taiga #5747](https://tree.taiga.io/project/penpot/issue/5747)
|
||||
- When editing text if font is changed, the proportions of the rendered shape are wrong [Taiga #5786](https://tree.taiga.io/project/penpot/issue/5786)
|
||||
|
||||
## 1.19.2
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
- Navigate up in layer hierarchy with Shift+Enter shortcut [Taiga #5734](https://tree.taiga.io/project/penpot/us/5734)
|
||||
- Click on the flow tags open viewer with the selected frame [Taiga #5044](https://tree.taiga.io/project/penpot/us/5044)
|
||||
- Add Dutch language & update translation files with weblate
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix unexpected output on get-page rpc method when invalid object-id is provided [Github #3546](https://github.com/penpot/penpot/issues/3546)
|
||||
- Fix Invalid files amount after moving file from Project to Drafts [Taiga #5638](https://tree.taiga.io/project/penpot/us/5638)
|
||||
- Fix deleted pages comments shown in right sidebar [Taiga #5648](https://tree.taiga.io/project/penpot/us/5648)
|
||||
- Fix tooltip on toggle visibility and toggle lock buttons [Taiga #5141](https://tree.taiga.io/project/penpot/issue/5141)
|
||||
|
||||
|
||||
## 1.19.1
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix components not registered as updated [Taiga #5725](https://tree.taiga.io/project/penpot/issue/5725)
|
||||
|
||||
## 1.19.0
|
||||
|
||||
### :boom: Breaking changes & Deprecations
|
||||
|
||||
### :sparkles: New features
|
||||
|
||||
- Default naming of text layers [Taiga #2836](https://tree.taiga.io/project/penpot/us/2836)
|
||||
- Create typography style from a selected text layer [Taiga #3041](https://tree.taiga.io/project/penpot/us/3041)
|
||||
- Create typography style from a selected text layer[Taiga #3041](https://tree.taiga.io/project/penpot/us/3041)
|
||||
- Board as ruler origin [Taiga #4833](https://tree.taiga.io/project/penpot/us/4833)
|
||||
- Access tokens support [Taiga #4460](https://tree.taiga.io/project/penpot/us/4460)
|
||||
- Show interactions setting at the view mode [Taiga #1330](https://tree.taiga.io/project/penpot/issue/1330)
|
||||
- Improve dashboard performance related to thumbnails; now the thumbnails are
|
||||
rendered as bitmap images.
|
||||
- Add the ability to disable google fonts provider with the `disable-google-fonts-provider` flag
|
||||
- Add the ability to disable dashboard templates section with the `disable-dashboard-templates-section` flag
|
||||
- Add the ability to use the registration whitelist with OICD [Github #3348](https://github.com/penpot/penpot/issues/3348)
|
||||
- Add support for local caching of google fonts (this avoids exposing the final user IP to
|
||||
goolge and reduces the amount of request sent to google)
|
||||
- Set smooth/instant autoscroll depending on distance [GitHub #3377](https://github.com/penpot/penpot/issues/3377)
|
||||
- New component icon [Taiga #5290](https://tree.taiga.io/project/penpot/us/5290)
|
||||
- Show a confirmation dialog when an user tries to publish an empty library [Taiga #5294](https://tree.taiga.io/project/penpot/us/5294)
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix files can be opened from multiple urls [Taiga #5310](https://tree.taiga.io/project/penpot/issue/5310)
|
||||
- Fix asset color item was created from the selected layer [Taiga #5180](https://tree.taiga.io/project/penpot/issue/5180)
|
||||
- Fix unpublish more than one library at the same time [Taiga #5532](https://tree.taiga.io/project/penpot/issue/5532)
|
||||
- Fix drag projects on dahsboard [Taiga #5531](https://tree.taiga.io/project/penpot/issue/5531)
|
||||
- Fix allow team name to be all blank [Taiga #5527](https://tree.taiga.io/project/penpot/issue/5527)
|
||||
- Fix search font visualitation [Taiga #5523](https://tree.taiga.io/project/penpot/issue/5523)
|
||||
- Fix create and account only with spaces [Taiga #5518](https://tree.taiga.io/project/penpot/issue/5518)
|
||||
- Fix context menu outside screen [Taiga #5524](https://tree.taiga.io/project/penpot/issue/5524)
|
||||
- Fix graphic item rename on assets pannel [Taiga #5556](https://tree.taiga.io/project/penpot/issue/5556)
|
||||
- Fix component and media name validation on assets panel [Taiga #5555](https://tree.taiga.io/project/penpot/issue/5555)
|
||||
- Fix problem with selection shortcuts [Taiga #5492](https://tree.taiga.io/project/penpot/issue/5492)
|
||||
- Fix issue with paths line to curve and concurrent editing [Taiga #5191](https://tree.taiga.io/project/penpot/issue/5191)
|
||||
- Fix problems with locked layers [Taiga #5139](https://tree.taiga.io/project/penpot/issue/5139)
|
||||
- Fix export from shared prototype [Taiga #5565](https://tree.taiga.io/project/penpot/issue/5565)
|
||||
- Fix email change: validation error displaying even after both fields are identical [Taiga #5514](https://tree.taiga.io/project/penpot/issue/5514)
|
||||
- Fix scroll on viewer comment list [Taiga #5563](https://tree.taiga.io/project/penpot/issue/5563)
|
||||
- Fix context menu z-index [Taiga #5561](https://tree.taiga.io/project/penpot/issue/5561)
|
||||
- Fix select all checkbox on shared link config [Taiga #5566](https://tree.taiga.io/project/penpot/issue/5566)
|
||||
- Fix validation on full name input on account creation [Taiga #5516](https://tree.taiga.io/project/penpot/issue/5516)
|
||||
- Fix validation on team name input [Taiga #5510](https://tree.taiga.io/project/penpot/issue/5510)
|
||||
- Fix incorrect uri generation issues on share-link modal [Taiga #5564](https://tree.taiga.io/project/penpot/issue/5564)
|
||||
- Fix cache issues with share-links [Taiga #5559](https://tree.taiga.io/project/penpot/issue/5559)
|
||||
- Makes height priority for the rows/columns grids [#2774](https://github.com/penpot/penpot/issues/2774)
|
||||
- Fix problem with comments mode not staying [#3363](https://github.com/penpot/penpot/issues/3363)
|
||||
- Fix problem with comments when user left the team [Taiga #5562](https://tree.taiga.io/project/penpot/issue/5562)
|
||||
- Fix problem with images patterns repeating [#3372](https://github.com/penpot/penpot/issues/3372)
|
||||
- Fix grid not being clipped in frames [#3365](https://github.com/penpot/penpot/issues/3365)
|
||||
- Fix cut/delete text layer when while creating text [Taiga #5602](https://tree.taiga.io/project/penpot/issue/5602)
|
||||
- Fix picking a gradient color in recent colors for a new color in the assets tab [Taiga #5601](https://tree.taiga.io/project/penpot/issue/5601)
|
||||
- Fix problem with importation process [Taiga #5597](https://tree.taiga.io/project/penpot/issue/5597)
|
||||
- Fix problem with HSV color picker [#3317](https://github.com/penpot/penpot/issues/3317)
|
||||
- Fix problem with slashes in layers names for exporter [#3276](https://github.com/penpot/penpot/issues/3276)
|
||||
- Fix incorrect modified data on moving files on dashboard [Taiga #5530](https://tree.taiga.io/project/penpot/issue/5530)
|
||||
- Fix focus handling on comments edition [Taiga #5560](https://tree.taiga.io/project/penpot/issue/5560)
|
||||
- Fix incorrect fullname use on registring user after OIDC authentication [Taiga #5517](https://tree.taiga.io/project/penpot/issue/5517)
|
||||
- Fix incorrect modified-at on project after import file [Taiga #5268](https://tree.taiga.io/project/penpot/issue/5268)
|
||||
- Fix incorrect message after sending invitation to already member [Taiga 5599](https://tree.taiga.io/project/penpot/issue/5599)
|
||||
- Fix text decoration on button [Taiga #5301](https://tree.taiga.io/project/penpot/issue/5301)
|
||||
- Fix menu order on design tab [Taiga #5195](https://tree.taiga.io/project/penpot/issue/5195)
|
||||
- Fix search bar width on layer tab [Taiga #5445](https://tree.taiga.io/project/penpot/issue/5445)
|
||||
- Fix border radius values with decimals [Taiga #5283](https://tree.taiga.io/project/penpot/issue/5283)
|
||||
- Fix shortcuts translations not homogenized [Taiga #5141](https://tree.taiga.io/project/penpot/issue/5141)
|
||||
- Fix overlay manual position in nested boards [Taiga #5135](https://tree.taiga.io/project/penpot/issue/5135)
|
||||
- Fix close overlay from a nested board [Taiga #5587](https://tree.taiga.io/project/penpot/issue/5587)
|
||||
- Fix overlay position when it has shadow or blur [Taiga #4752](https://tree.taiga.io/project/penpot/issue/4752)
|
||||
- Fix overlay position when there are elements fixed when scrolling [Taiga #4383](https://tree.taiga.io/project/penpot/issue/4383)
|
||||
- Fix problem when sliding color picker in selected-colors [#3150](https://github.com/penpot/penpot/issues/3150)
|
||||
- Fix error screen on upload image error [Taiga #5608](https://tree.taiga.io/project/penpot/issue/5608)
|
||||
- Fix bad frame-id for certain componentes [#3205](https://github.com/penpot/penpot/issues/3205)
|
||||
- Fix paste elements at bottom of frame [Taig #5253](https://tree.taiga.io/project/penpot/issue/5253)
|
||||
- Fix new-file button on project not redirecting to the new file [Taiga #5610](https://tree.taiga.io/project/penpot/issue/5610)
|
||||
- Fix retrieve user comments in dashboard [Taiga #5607](https://tree.taiga.io/project/penpot/issue/5607)
|
||||
- Locks shapes when moved inside a locked parent [Taiga #5252](https://tree.taiga.io/project/penpot/issue/5252)
|
||||
- Fix rotate several elements in bulk [Taiga #5165](https://tree.taiga.io/project/penpot/issue/5165)
|
||||
- Fix onboarding slides height [Taiga #5373](https://tree.taiga.io/project/penpot/issue/5373)
|
||||
- Fix create typography with section closed [Taiga #5574](https://tree.taiga.io/project/penpot/issue/5574)
|
||||
- Fix exports menu on viewer mode [Taiga #5568](https://tree.taiga.io/project/penpot/issue/5568)
|
||||
- Fix create empty comments [Taiga #5536](https://tree.taiga.io/project/penpot/issue/5536)
|
||||
- Fix text changes not propagated to copy [Taiga #5364](https://tree.taiga.io/project/penpot/issue/5364)
|
||||
- Fix position of text cursor is a bit too high in Invitations section [Taiga #5511](https://tree.taiga.io/project/penpot/issue/5511)
|
||||
- Fix undo when updating several texts [Taiga #5197](https://tree.taiga.io/project/penpot/issue/5197)
|
||||
- Fix assets right click button for multiple selection [Taiga #5545](https://tree.taiga.io/project/penpot/issue/5545)
|
||||
- Fix problem with precision in resizes [Taiga #5623](https://tree.taiga.io/project/penpot/issue/5623)
|
||||
- Fix absolute positioned layouts not showing flex properties [Taiga #5630](https://tree.taiga.io/project/penpot/issue/5630)
|
||||
- Fix text gradient handlers [Taiga #4047](https://tree.taiga.io/project/penpot/issue/4047)
|
||||
- Fix when user deletes one file during import it is impossible to finish importing of second file [Taiga #5656](https://tree.taiga.io/project/penpot/issue/5656)
|
||||
- Fix export multiple images when only one of them has export settings [Taiga #5649](https://tree.taiga.io/project/penpot/issue/5649)
|
||||
- Fix error when a user different than the thread creator edits a comment [Taiga #5647](https://tree.taiga.io/project/penpot/issue/5647)
|
||||
- Fix unnecessary button [Taiga #3312](https://tree.taiga.io/project/penpot/issue/3312)
|
||||
- Fix copy color information in several formats [Taiga #4723](https://tree.taiga.io/project/penpot/issue/4723)
|
||||
- Fix dropdown width [Taiga #5541](https://tree.taiga.io/project/penpot/issue/5541)
|
||||
- Fix enable comment mode and insert image keeps on comment mode [Taiga #5678](https://tree.taiga.io/project/penpot/issue/5678)
|
||||
- Fix enable undo just after using pencil [Taiga #5674](https://tree.taiga.io/project/penpot/issue/5674)
|
||||
- Fix 400 error when user changes password [Taiga #5643](https://tree.taiga.io/project/penpot/issue/5643)
|
||||
- Fix cannot undo layer styles [Taiga #5676](https://tree.taiga.io/project/penpot/issue/5676)
|
||||
- Fix unexpected exception on boolean shapes [Taiga #5685](https://tree.taiga.io/project/penpot/issue/5685)
|
||||
- Fix ctrl+z on select not working [Taiga #5677](https://tree.taiga.io/project/penpot/issue/5677)
|
||||
- Fix thubmnail rendering flashing [Taiga #5675](https://tree.taiga.io/project/penpot/issue/5675)
|
||||
|
||||
### :arrow_up: Deps updates
|
||||
|
||||
- Update google fonts catalog (at 2023/07/06) [Taiga #5592](https://tree.taiga.io/project/penpot/issue/5592)
|
||||
|
||||
|
||||
### :heart: Community contributions by (Thank you!)
|
||||
|
||||
- Update Typography palette order (by @akshay-gupta7) [Github #3156](https://github.com/penpot/penpot/pull/3156)
|
||||
- Palettes (color, typographies) empty state (by @akshay-gupta7) [Github #3160](https://github.com/penpot/penpot/pull/3160)
|
||||
- Duplicate objects via drag + alt (by @akshay-gupta7) [Github #3147](https://github.com/penpot/penpot/pull/3147)
|
||||
@@ -205,7 +31,6 @@
|
||||
- Open project in new tab from workspace (by @akshay-gupta7) [Github #3246](https://github.com/penpot/penpot/pull/3246)
|
||||
- Distribute fix enabled when two elements were selected (by @dfelinto) [Github #3266](https://github.com/penpot/penpot/pull/3266)
|
||||
- Distribute vertical spacing failing for overlapped text (by @dfelinto) [Github #3267](https://github.com/penpot/penpot/pull/3267)
|
||||
- bug Change independent corner radius input tooltips #3332 (by @astudentinearth) [Github #3332](https://github.com/penpot/penpot/pull/3332)
|
||||
|
||||
## 1.18.6
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||

|
||||
|
||||
🎇 **Penpot Fest exceeded all expectations - it was a complete success!** 🎇 Penpot Fest is our first Design event that brought designers and developers from the Open Source communities and beyond. Watch the replay of the talks on our [Youtube channel](https://www.youtube.com/playlist?list=PLgcCPfOv5v56-fghJo2dHNBqL9zlDTslh) or [Peertube channel](https://peertube.kaleidos.net/w/p/1tWgyJTt8sKbWwCEcBimZW)
|
||||
**:tada: [Important Notice!] :tada:** Our very first **Penpot Fest** is happening on June 28-30, Barcelona (Spain). **Secure yourself a ticket** to know everything about the present and future of Penpot and be part of the conversation! See details on the amazing venue and speakers lineup at [penpotfest.org](https://penpotfest.org)! :zap:
|
||||
|
||||
Penpot is the first **Open Source** design and prototyping platform meant for cross-domain teams. Non dependent on operating systems, Penpot is web based and works with open standards (SVG). Penpot invites designers all over the world to fall in love with open source while getting developers excited about the design process in return.
|
||||
|
||||
@@ -109,7 +109,6 @@ Every sort of contribution will be very helpful to enhance Penpot. How you’ll
|
||||
- Create and [share Libraries & templates](https://penpot.app/libraries-templates.html) that will be helpful for the community
|
||||
- Become a [translator](https://help.penpot.app/contributing-guide/translations)
|
||||
- Give feedback: [Mail us](mailto:support@penpot.app)
|
||||
- **Contribute to Penpot's code:** [Watch this video](https://www.youtube.com/watch?v=TpN0osiY-8k) by Alejandro Alonso, CIO and developer at Penpot, where he gives us a hands-on demo of how to use Penpot’s repository and make changes in both front and back end
|
||||
|
||||
To find (almost) everything you need to know on how to contribute to Penpot, refer to the [contributing-guide](https://help.penpot.app/contributing-guide/).
|
||||
|
||||
|
||||
7
backend/.gitignore
vendored
7
backend/.gitignore
vendored
@@ -1,7 +0,0 @@
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
@@ -3,10 +3,10 @@
|
||||
|
||||
:deps
|
||||
{penpot/common {:local/root "../common"}
|
||||
org.clojure/clojure {:mvn/version "1.12.0-alpha5"}
|
||||
org.clojure/tools.namespace {:mvn/version "1.4.4"}
|
||||
org.clojure/clojure {:mvn/version "1.11.1"}
|
||||
org.clojure/core.async {:mvn/version "1.6.673"}
|
||||
|
||||
com.github.luben/zstd-jni {:mvn/version "1.5.5-10"}
|
||||
com.github.luben/zstd-jni {:mvn/version "1.5.2-5"}
|
||||
|
||||
io.prometheus/simpleclient {:mvn/version "0.16.0"}
|
||||
io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"}
|
||||
@@ -17,32 +17,30 @@
|
||||
|
||||
io.prometheus/simpleclient_httpserver {:mvn/version "0.16.0"}
|
||||
|
||||
io.lettuce/lettuce-core {:mvn/version "6.2.6.RELEASE"}
|
||||
io.lettuce/lettuce-core {:mvn/version "6.2.2.RELEASE"}
|
||||
java-http-clj/java-http-clj {:mvn/version "0.4.3"}
|
||||
|
||||
funcool/yetti
|
||||
{:git/tag "v10.0"
|
||||
:git/sha "520613f"
|
||||
{:git/tag "v9.15"
|
||||
:git/sha "aa9b967"
|
||||
:git/url "https://github.com/funcool/yetti.git"
|
||||
:exclusions [org.slf4j/slf4j-api]}
|
||||
|
||||
com.github.seancorfield/next.jdbc {:mvn/version "1.3.894"}
|
||||
metosin/reitit-core {:mvn/version "0.6.0"}
|
||||
nrepl/nrepl {:mvn/version "1.1.0"}
|
||||
cider/cider-nrepl {:mvn/version "0.43.1"}
|
||||
com.github.seancorfield/next.jdbc {:mvn/version "1.3.847"}
|
||||
metosin/reitit-core {:mvn/version "0.5.18"}
|
||||
|
||||
org.postgresql/postgresql {:mvn/version "42.6.0"}
|
||||
|
||||
com.zaxxer/HikariCP {:mvn/version "5.1.0"}
|
||||
com.zaxxer/HikariCP {:mvn/version "5.0.1"}
|
||||
|
||||
io.whitfin/siphash {:mvn/version "2.0.0"}
|
||||
|
||||
buddy/buddy-hashers {:mvn/version "2.0.167"}
|
||||
buddy/buddy-sign {:mvn/version "3.5.351"}
|
||||
buddy/buddy-hashers {:mvn/version "1.8.158"}
|
||||
buddy/buddy-sign {:mvn/version "3.4.333"}
|
||||
|
||||
com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.8"}
|
||||
com.github.ben-manes.caffeine/caffeine {:mvn/version "3.1.5"}
|
||||
|
||||
org.jsoup/jsoup {:mvn/version "1.16.2"}
|
||||
org.jsoup/jsoup {:mvn/version "1.15.3"}
|
||||
org.im4java/im4java
|
||||
{:git/tag "1.4.0-penpot-2"
|
||||
:git/sha "e2b3e16"
|
||||
@@ -51,13 +49,14 @@
|
||||
org.lz4/lz4-java {:mvn/version "1.8.0"}
|
||||
|
||||
org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
|
||||
integrant/integrant {:mvn/version "0.8.0"}
|
||||
|
||||
dawran6/emoji {:mvn/version "0.1.5"}
|
||||
markdown-clj/markdown-clj {:mvn/version "1.11.7"}
|
||||
markdown-clj/markdown-clj {:mvn/version "1.11.4"}
|
||||
|
||||
;; Pretty Print specs
|
||||
pretty-spec/pretty-spec {:mvn/version "0.1.4"}
|
||||
software.amazon.awssdk/s3 {:mvn/version "2.20.138"}
|
||||
software.amazon.awssdk/s3 {:mvn/version "2.19.29"}
|
||||
}
|
||||
|
||||
:paths ["src" "resources" "target/classes"]
|
||||
@@ -65,6 +64,7 @@
|
||||
{:dev
|
||||
{:extra-deps
|
||||
{com.bhauman/rebel-readline {:mvn/version "RELEASE"}
|
||||
org.clojure/tools.namespace {:mvn/version "RELEASE"}
|
||||
clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
|
||||
org.clojure/data.csv {:mvn/version "RELEASE"}
|
||||
com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
|
||||
@@ -73,7 +73,7 @@
|
||||
|
||||
:build
|
||||
{:extra-deps
|
||||
{io.github.clojure/tools.build {:git/tag "v0.9.5" :git/sha "24f2894"}}
|
||||
{io.github.clojure/tools.build {:git/tag "v0.9.3" :git/sha "e537cd1"}}
|
||||
:ns-default build}
|
||||
|
||||
:test
|
||||
|
||||
@@ -19,12 +19,10 @@
|
||||
[app.common.schema.generators :as sg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.transit :as t]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.config :as cfg]
|
||||
[app.main :as main]
|
||||
[app.srepl.helpers :as srepl.helpers]
|
||||
[app.srepl.helpers]
|
||||
[app.srepl.main :as srepl]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.json :as json]
|
||||
@@ -50,12 +48,13 @@
|
||||
[malli.generator :as mg]
|
||||
[malli.registry :as mr]
|
||||
[malli.transform :as mt]
|
||||
[malli.util :as mu]
|
||||
[promesa.exec :as px]))
|
||||
[malli.util :as mu]))
|
||||
|
||||
(repl/disable-reload! (find-ns 'integrant.core))
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defonce system nil)
|
||||
|
||||
;; --- Benchmarking Tools
|
||||
|
||||
(defmacro run-quick-bench
|
||||
@@ -93,14 +92,20 @@
|
||||
(defn- start
|
||||
[]
|
||||
(try
|
||||
(main/start)
|
||||
(alter-var-root #'system (fn [sys]
|
||||
(when sys (ig/halt! sys))
|
||||
(-> (merge main/system-config main/worker-config)
|
||||
(ig/prep)
|
||||
(ig/init))))
|
||||
:started
|
||||
(catch Throwable cause
|
||||
(ex/print-throwable cause))))
|
||||
|
||||
(defn- stop
|
||||
[]
|
||||
(main/stop)
|
||||
(alter-var-root #'system (fn [sys]
|
||||
(when sys (ig/halt! sys))
|
||||
nil))
|
||||
:stopped)
|
||||
|
||||
(defn restart
|
||||
@@ -113,26 +118,62 @@
|
||||
(stop)
|
||||
(repl/refresh-all :after 'user/start))
|
||||
|
||||
;; (defn compression-bench
|
||||
;; [data]
|
||||
;; (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
|
||||
;; v1 (time (humanize (alength (blob/encode data {:version 1}))))
|
||||
;; v3 (time (humanize (alength (blob/encode data {:version 3}))))
|
||||
;; v4 (time (humanize (alength (blob/encode data {:version 4}))))
|
||||
;; v5 (time (humanize (alength (blob/encode data {:version 5}))))
|
||||
;; v6 (time (humanize (alength (blob/encode data {:version 6}))))
|
||||
;; ]
|
||||
;; (print-table
|
||||
;; [{
|
||||
;; :v1 v1
|
||||
;; :v3 v3
|
||||
;; :v4 v4
|
||||
;; :v5 v5
|
||||
;; :v6 v6
|
||||
;; }])))
|
||||
(defn compression-bench
|
||||
[data]
|
||||
(let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))
|
||||
v1 (time (humanize (alength (blob/encode data {:version 1}))))
|
||||
v3 (time (humanize (alength (blob/encode data {:version 3}))))
|
||||
v4 (time (humanize (alength (blob/encode data {:version 4}))))
|
||||
v5 (time (humanize (alength (blob/encode data {:version 5}))))
|
||||
v6 (time (humanize (alength (blob/encode data {:version 6}))))
|
||||
]
|
||||
(print-table
|
||||
[{
|
||||
:v1 v1
|
||||
:v3 v3
|
||||
:v4 v4
|
||||
:v5 v5
|
||||
:v6 v6
|
||||
}])))
|
||||
|
||||
(defonce debug-tap
|
||||
(do
|
||||
(add-tap #(locking debug-tap
|
||||
(prn "tap debug:" %)))
|
||||
1))
|
||||
|
||||
|
||||
(sm/def! ::test
|
||||
[:map {:title "Foo"}
|
||||
[:x :int]
|
||||
[:y {:min 0} :double]
|
||||
[:bar
|
||||
[:map {:title "Bar"}
|
||||
[:z :string]
|
||||
[:v ::sm/uuid]]]
|
||||
[:items
|
||||
[:vector ::dt/instant]]])
|
||||
|
||||
(sm/def! ::test2
|
||||
[:multi {:title "Foo" :dispatch :type}
|
||||
[:x
|
||||
[:map {:title "FooX"}
|
||||
[:type [:= :x]]
|
||||
[:x :int]]]
|
||||
[:y
|
||||
[:map
|
||||
[:type [:= :x]]
|
||||
[:y [::sm/one-of #{:a :b :c}]]]]
|
||||
[:z
|
||||
[:map {:title "FooZ"}
|
||||
[:z
|
||||
[:multi {:title "Bar" :dispatch :type}
|
||||
[:a
|
||||
[:map
|
||||
[:type [:= :a]]
|
||||
[:a :int]]]
|
||||
[:b
|
||||
[:map
|
||||
[:type [:= :b]]
|
||||
[:b :int]]]]]]]])
|
||||
|
||||
|
||||
@@ -1,26 +1,18 @@
|
||||
{
|
||||
"name": "backend",
|
||||
"version": "1.0.0",
|
||||
"license": "MPL-2.0",
|
||||
"author": "Kaleidos INC",
|
||||
"private": true,
|
||||
"packageManager": "yarn@4.0.2",
|
||||
"name": "uxbox-back",
|
||||
"version": "0.1.0",
|
||||
"description": "The Open-Source prototyping tool",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"build-emails": "./scripts/build-email-templates.sh"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/penpot/penpot"
|
||||
},
|
||||
"dependencies": {
|
||||
"luxon": "^3.4.2",
|
||||
"sax": "^1.2.4"
|
||||
"url": "git+https://github.com/uxbox/uxbox.git"
|
||||
},
|
||||
"author": "Uxbox",
|
||||
"license": "SEE LICENSE IN <LICENSE>",
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.1",
|
||||
"source-map-support": "^0.5.21",
|
||||
"ws": "^8.13.0"
|
||||
},
|
||||
"scripts": {
|
||||
"fmt:clj:check": "cljfmt check --parallel=false src/ test/",
|
||||
"fmt:clj": "cljfmt fix --parallel=true src/ test/",
|
||||
"lint:clj": "clj-kondo --parallel --lint src/"
|
||||
"mjml": "^4.6.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,30 +1,36 @@
|
||||
[{:id "material-design-3"
|
||||
:name "Material Design 3"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-md3.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/main/Material%20Design%203.penpot"}
|
||||
{:id "tutorial-for-beginners"
|
||||
:name "Tutorial for beginners"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/tutorial-for-beginners.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/tutorial-for-beginners.penpot"}
|
||||
{:id "penpot-design-system"
|
||||
:name "Penpot Design System"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-ds-penpot.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Penpot-Design-system.penpot"}
|
||||
{:id "flex-layout-playground"
|
||||
:name "Flex Layout Playground"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/main/Flex%20Layout%20Playground.penpot"}
|
||||
{:id "wireframing-kit"
|
||||
:name "Wireframing Kit"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-wireframes.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/wireframing-kit.penpot"}
|
||||
{:id "ant-design"
|
||||
:name "Ant Design UI Kit (lite)"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-ant-design.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Ant-Design-UI-Kit-Lite.penpot"}
|
||||
{:id "cocomaterial"
|
||||
:name "Cocomaterial"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-cocomaterial.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Cocomaterial.penpot"}
|
||||
{:id "circum-icons"
|
||||
:name "Circum Icons pack"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-circum.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/CircumIcons.penpot"}
|
||||
{:id "coreui"
|
||||
:name "CoreUI"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-coreui.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/main/CoreUI%20DesignSystem%20(DEMO).penpot"}
|
||||
{:id "whiteboarding-kit"
|
||||
:name "Whiteboarding Kit"
|
||||
:thumbnail-uri "https://penpot.app/images/libraries/cover-whiteboards.jpg"
|
||||
:file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Whiteboarding-mapping-kit.penpot"}]
|
||||
|
||||
@@ -22,13 +22,11 @@
|
||||
{% endif %}
|
||||
{% if item.params-schema-js %}
|
||||
<span class="tag">
|
||||
<span>SCHEMA</span>
|
||||
<span>SC</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
|
||||
{% if item.sse %}
|
||||
{% else %}
|
||||
<span class="tag">
|
||||
<span>SSE</span>
|
||||
<span>SP</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
@@ -156,7 +156,7 @@ h4 {
|
||||
}
|
||||
|
||||
.rpc-row-info > .module {
|
||||
width: 150px;
|
||||
width: 120px;
|
||||
font-weight: bold;
|
||||
border-right: 1px dotted #777;
|
||||
text-align: right;
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
<h2>GENERAL NOTES</h2>
|
||||
|
||||
<h3>Authentication</h3>
|
||||
<p>The penpot backend right now offers two way for authenticate the request:
|
||||
<p>The penpot backend right now offerts two way for authenticate the request:
|
||||
<b>cookies</b> (the same mechanism that we use ourselves on accessing the API from the
|
||||
web application) and <b>access tokens</b>.</p>
|
||||
|
||||
@@ -56,17 +56,6 @@
|
||||
desired content-type on the <b>`Accept`</b> header, the transit encoding is used
|
||||
by default.</p>
|
||||
|
||||
<h3>SSE (Server-Sent Events)</h3>
|
||||
<p>The methods marked with <b>SSE</b> returns
|
||||
a <a href="https://html.spec.whatwg.org/multipage/server-sent-events.html"> SSE
|
||||
formatted</a> stream on the response body, always with status 200. The events are
|
||||
always encoded using `application/transit+json` encoding (for now no content
|
||||
negotiation is possible on methods that return SSE streams). </p>
|
||||
<p>On the javascript side you can use
|
||||
the <a href="https://github.com/rexxars/eventsource-parser">eventsoure-parser</a>
|
||||
library for propertly parsing the response body using the
|
||||
standard <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
|
||||
API</a></p>
|
||||
|
||||
<h3>Limits</h3>
|
||||
<p>The rate limit work per user basis (this means that different api keys share
|
||||
|
||||
@@ -6,17 +6,11 @@ Debug Main Page
|
||||
|
||||
{% block content %}
|
||||
<nav>
|
||||
<div class="title">
|
||||
<h1>ADMIN DEBUG INTERFACE</h1>
|
||||
</div>
|
||||
<h1>Debug INDEX:</h1>
|
||||
<div>[<a href="/dbg/error">ERRORS</a>]</div>
|
||||
</nav>
|
||||
<main class="dashboard">
|
||||
<main class="index">
|
||||
<section class="widget">
|
||||
<fieldset>
|
||||
<legend>Error reports</legend>
|
||||
<desc><a href="/dbg/error">CLICK HERE TO SEE THE ERROR REPORTS</a> </desc>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Download file data:</legend>
|
||||
<desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
|
||||
@@ -43,42 +37,9 @@ Debug Main Page
|
||||
<input type="checkbox" name="reuseid" />
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<input type="submit" value="Upload" />
|
||||
</div>
|
||||
<input type="submit" value="Upload" />
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Profile Management</legend>
|
||||
<form method="post" action="/dbg/actions/resend-email-verification">
|
||||
<div class="row">
|
||||
<input type="email" name="email" placeholder="example@example.com" value="" />
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<label for="force-verify">Are you sure?</label>
|
||||
<input id="force-verify" type="checkbox" name="force" />
|
||||
<br />
|
||||
<small>
|
||||
This is a just a security double check for prevent non intentional submits.
|
||||
</small>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<input type="submit" name="resend" value="Resend Verification" />
|
||||
<input type="submit" name="verify" value="Verify" />
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<input type="submit" class="danger" name="block" value="Block" />
|
||||
<input type="submit" class="danger" name="unblock" value="Unblock" />
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
|
||||
|
||||
|
||||
</section>
|
||||
|
||||
<section class="widget">
|
||||
@@ -162,35 +123,5 @@ Debug Main Page
|
||||
</form>
|
||||
</fieldset>
|
||||
</section>
|
||||
|
||||
<section class="widget">
|
||||
<fieldset>
|
||||
<legend>Reset file version</legend>
|
||||
<desc>Allows reset file data version to a specific number/</desc>
|
||||
|
||||
<form method="post" action="/dbg/actions/reset-file-data-version">
|
||||
<div class="row">
|
||||
<input type="text" style="width:300px" name="file-id" placeholder="file-id" />
|
||||
</div>
|
||||
<div class="row">
|
||||
<input type="number" style="width:100px" name="version" placeholder="version" value="32" />
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<label for="force-version">Are you sure?</label>
|
||||
<input id="force-version" type="checkbox" name="force" />
|
||||
<br />
|
||||
<small>
|
||||
This is a just a security double check for prevent non intentional submits.
|
||||
</small>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="row">
|
||||
<input type="submit" value="Submit" />
|
||||
</div>
|
||||
</form>
|
||||
</fieldset>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
|
||||
@@ -6,19 +6,13 @@ penpot - error list
|
||||
|
||||
{% block content %}
|
||||
<nav>
|
||||
<div class="title">
|
||||
<h1>Error reports (last 200)</h1>
|
||||
</div>
|
||||
<h1>Latest error reports:</h1>
|
||||
</nav>
|
||||
<main class="horizontal-list">
|
||||
<ul>
|
||||
{% for item in items %}
|
||||
<li>
|
||||
<a class="date" href="/dbg/error/{{item.id}}">{{item.created-at}}</a>
|
||||
<a class="hint" href="/dbg/error/{{item.id}}">
|
||||
<span class="title">{{item.hint|abbreviate:150}}</span>
|
||||
</a>
|
||||
</li>
|
||||
<li><a class="date" href="/dbg/error/{{item.id}}">{{item.created-at}}</a>
|
||||
<span class="title">{{item.hint|abbreviate:150}}</span></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</main>
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{% extends "app/templates/base.tmpl" %}
|
||||
|
||||
{% block title %}
|
||||
Report: {{hint|abbreviate:150}} - {{id}} - Penpot Error Report (v3)
|
||||
penpot - error report v2 {{id}}
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<nav>
|
||||
<div>[<a href="/dbg/error">⮜</a>]</div>
|
||||
<div>[<a href="#head">head</a>]</div>
|
||||
<div>[<a href="#message">message</a>]</div>
|
||||
<div>[<a href="#props">props</a>]</div>
|
||||
<div>[<a href="#context">context</a>]</div>
|
||||
{% if params %}
|
||||
@@ -29,11 +29,10 @@ Report: {{hint|abbreviate:150}} - {{id}} - Penpot Error Report (v3)
|
||||
<main>
|
||||
<div class="table">
|
||||
<div class="table-row multiline">
|
||||
<div id="head" class="table-key">HEAD</div>
|
||||
<div id="message" class="table-key">MESSAGE: </div>
|
||||
|
||||
<div class="table-val">
|
||||
<h1><span class="not-important">Hint:</span> <br/> {{hint}}</h1>
|
||||
<h2><span class="not-important">Reported at:</span> <br/> {{created-at}}</h2>
|
||||
<h2><span class="not-important">Report ID:</span> <br/> {{id}}</h2>
|
||||
<h1>{{hint}}</h1>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -72,7 +71,7 @@ Report: {{hint|abbreviate:150}} - {{id}} - Penpot Error Report (v3)
|
||||
|
||||
{% if value %}
|
||||
<div class="table-row multiline">
|
||||
<div id="value" class="table-key">VALUE: </div>
|
||||
<div id="value" class="table-key">VALIDATION VALUE: </div>
|
||||
<div class="table-val">
|
||||
<pre>{{value}}</pre>
|
||||
</div>
|
||||
|
||||
@@ -36,11 +36,6 @@ small {
|
||||
color: #888;
|
||||
}
|
||||
|
||||
.not-important {
|
||||
color: #888;
|
||||
font-weight: 200;
|
||||
}
|
||||
|
||||
small > strong {
|
||||
font-size: 9px;
|
||||
}
|
||||
@@ -55,13 +50,7 @@ nav {
|
||||
background: #e3e3e3;
|
||||
}
|
||||
|
||||
nav > .title {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
nav > .title > h1 {
|
||||
nav > h1 {
|
||||
padding: 0px;
|
||||
margin: 0px;
|
||||
font-size: 11px;
|
||||
@@ -116,50 +105,29 @@ nav > div:not(:last-child) {
|
||||
width: unset;
|
||||
}
|
||||
|
||||
.dashboard {
|
||||
.index {
|
||||
margin-top: 40px;
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.widget {
|
||||
.index > section {
|
||||
padding: 10px;
|
||||
background-color: #e3e3e3;
|
||||
max-width: 400px;
|
||||
margin: 5px;
|
||||
height: fit-content;
|
||||
}
|
||||
|
||||
.widget input[type=submit] {
|
||||
outline: none;
|
||||
border: 1px solid gray;
|
||||
border-radius: 2px;
|
||||
padding: 3px 5px;
|
||||
}
|
||||
|
||||
.widget input[type=submit].danger {
|
||||
outline: none;
|
||||
border: 1px solid red;
|
||||
border-radius: 2px;
|
||||
padding: 3px 5px;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.widget > fieldset {
|
||||
padding: 10px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
|
||||
.widget > fieldset:not(:last-child) {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.dashboard fieldset:not(:first-child) {
|
||||
.index fieldset:not(:first-child) {
|
||||
margin-top: 15px;
|
||||
}
|
||||
|
||||
/* .index > section:not(:last-child) { */
|
||||
/* margin-bottom: 10px; */
|
||||
/* } */
|
||||
|
||||
.widget > h2 {
|
||||
|
||||
.index > section > h2 {
|
||||
margin-top: 0px;
|
||||
}
|
||||
|
||||
@@ -183,6 +151,7 @@ nav > div:not(:last-child) {
|
||||
line-height: 18px;
|
||||
min-width: 210px;
|
||||
margin: 0px 20px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
@@ -3,17 +3,12 @@
|
||||
;; Optional: queue, ommited means Integer/MAX_VALUE
|
||||
;; Optional: timeout, ommited means no timeout
|
||||
;; Note: queue and timeout are excluding
|
||||
{:update-file/by-profile
|
||||
{:permits 1 :queue 5}
|
||||
{:update-file-by-id {:permits 1 :queue 3}
|
||||
:update-file {:permits 20}
|
||||
|
||||
:update-file/global {:permits 20}
|
||||
:derive-password {:permits 8}
|
||||
:process-font {:permits 4 :queue 32}
|
||||
:process-image {:permits 8 :queue 32}
|
||||
|
||||
:derive-password/global {:permits 8}
|
||||
:process-font/global {:permits 4}
|
||||
:process-image/global {:permits 8}
|
||||
|
||||
:file-thumbnail-ops/by-profile
|
||||
{:permits 2}
|
||||
|
||||
:submit-audit-events/by-profile
|
||||
:submit-audit-events-by-profile
|
||||
{:permits 1 :queue 3}}
|
||||
|
||||
@@ -22,15 +22,15 @@
|
||||
<Logger name="org.postgresql" level="error" />
|
||||
|
||||
<Logger name="app.rpc.commands.binfile" level="debug" />
|
||||
<Logger name="app.storage.tmp" level="info" />
|
||||
<Logger name="app.worker" level="trace" />
|
||||
<Logger name="app.storage.tmp" level="debug" />
|
||||
<Logger name="app.worker" level="info" />
|
||||
<Logger name="app.msgbus" level="info" />
|
||||
<Logger name="app.http.websocket" level="info" />
|
||||
<Logger name="app.util.websocket" level="info" />
|
||||
<Logger name="app.redis" level="info" />
|
||||
<Logger name="app.rpc.rlimit" level="info" />
|
||||
<Logger name="app.rpc.climit" level="info" />
|
||||
<Logger name="app.common.files.migrations" level="info" />
|
||||
<Logger name="app.rpc.mutations.files" level="info" />
|
||||
|
||||
<Logger name="app.loggers" level="debug" additivity="false">
|
||||
<AppenderRef ref="main" level="debug" />
|
||||
|
||||
@@ -13,14 +13,10 @@
|
||||
<Logger name="org.postgresql" level="error" />
|
||||
|
||||
<Logger name="app.util" level="info" />
|
||||
|
||||
<Logger name="app.loggers" level="debug" />
|
||||
|
||||
<Logger name="app" level="info" additivity="false">
|
||||
<AppenderRef ref="console" />
|
||||
</Logger>
|
||||
|
||||
|
||||
<Root level="info">
|
||||
<AppenderRef ref="console" />
|
||||
</Root>
|
||||
|
||||
@@ -18,8 +18,6 @@ cp scripts/manage.py target/dist/manage.py
|
||||
chmod +x target/dist/run.sh;
|
||||
chmod +x target/dist/manage.py
|
||||
|
||||
# Prefetch templates
|
||||
rm -rf builtin-templates;
|
||||
mkdir builtin-templates;
|
||||
# Prefetch
|
||||
bb ./scripts/prefetch-templates.clj resources/app/onboarding.edn builtin-templates/
|
||||
cp -r builtin-templates target/dist/
|
||||
|
||||
@@ -44,16 +44,11 @@ def send_eval(expr):
|
||||
s.send(b":repl/quit\n\n")
|
||||
|
||||
with s.makefile() as f:
|
||||
while True:
|
||||
line = f.readline()
|
||||
result = json.loads(line)
|
||||
tag = result.get("tag", None)
|
||||
if tag == "ret":
|
||||
return result.get("val", None), result.get("exception", None)
|
||||
elif tag == "out":
|
||||
print(result.get("val"), end="")
|
||||
else:
|
||||
raise RuntimeError("unexpected response from PREPL")
|
||||
result = json.load(f)
|
||||
tag = result.get("tag", None)
|
||||
if tag != "ret":
|
||||
raise RuntimeError("unexpected response from PREPL")
|
||||
return result.get("val", None), result.get("exception", None)
|
||||
|
||||
def encode(val):
|
||||
return json.dumps(json.dumps(val))
|
||||
@@ -65,7 +60,7 @@ def print_error(res):
|
||||
|
||||
def run_cmd(params):
|
||||
try:
|
||||
expr = "(app.srepl.cli/exec {})".format(encode(params))
|
||||
expr = "(app.srepl.ext/run-json-cmd {})".format(encode(params))
|
||||
res, failed = send_eval(expr)
|
||||
if failed:
|
||||
print_error(res)
|
||||
@@ -145,22 +140,12 @@ def derive_password(password):
|
||||
res = run_cmd(params)
|
||||
print(f"Derived password: \"{res}\"")
|
||||
|
||||
|
||||
def migrate_components_v2():
|
||||
params = {
|
||||
"cmd": "migrate-v2",
|
||||
"params": {}
|
||||
}
|
||||
|
||||
run_cmd(params)
|
||||
|
||||
available_commands = (
|
||||
"create-profile",
|
||||
"update-profile",
|
||||
"delete-profile",
|
||||
"search-profile",
|
||||
"derive-password",
|
||||
"migrate-components-v2",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -232,8 +217,3 @@ elif args.action == "search-profile":
|
||||
email = input("Email: ")
|
||||
|
||||
search_profile(email)
|
||||
|
||||
elif args.action == "migrate-components-v2":
|
||||
migrate_components_v2()
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
clojure -J-Xms50m -J-Xmx256m -J-XX:+UseSerialGC -Sdeps '{:deps {reply/reply {:mvn/version "0.5.0"}}}' -M -m reply.main --attach localhost:6064 -e "(in-ns 'app.main)"
|
||||
@@ -10,10 +10,9 @@ export PENPOT_FLAGS="\
|
||||
enable-login-with-google \
|
||||
enable-login-with-github \
|
||||
enable-login-with-gitlab \
|
||||
enable-backend-worker \
|
||||
enable-backend-asserts \
|
||||
enable-feature-fdata-pointer-map \
|
||||
enable-feature-fdata-objects-map \
|
||||
enable-fdata-storage-pointer-map \
|
||||
enable-fdata-storage-objets-map \
|
||||
enable-audit-log \
|
||||
enable-transit-readable-response \
|
||||
enable-demo-users \
|
||||
@@ -25,12 +24,7 @@ export PENPOT_FLAGS="\
|
||||
enable-rpc-rlimit \
|
||||
enable-soft-rpc-rlimit \
|
||||
enable-webhooks \
|
||||
enable-access-tokens \
|
||||
disable-feature-components-v2 \
|
||||
enable-file-validation \
|
||||
enable-file-schema-validation \
|
||||
disable-soft-file-schema-validation \
|
||||
disable-soft-file-validation";
|
||||
enable-access-tokens";
|
||||
|
||||
# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
|
||||
# export PENPOT_DATABASE_USERNAME="penpot"
|
||||
@@ -45,13 +39,10 @@ export PENPOT_FLAGS="\
|
||||
# export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
|
||||
|
||||
# Initialize MINIO config
|
||||
mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
|
||||
mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
|
||||
mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
|
||||
if [ "$?" = "1" ]; then
|
||||
mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
|
||||
fi
|
||||
mc mb penpot-s3/penpot -p -q
|
||||
mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
|
||||
mc admin user add penpot-s3 penpot-devenv penpot-devenv
|
||||
mc admin policy set penpot-s3 readwrite user=penpot-devenv
|
||||
mc mb penpot-s3/penpot -p
|
||||
|
||||
export AWS_ACCESS_KEY_ID=penpot-devenv
|
||||
export AWS_SECRET_ACCESS_KEY=penpot-devenv
|
||||
@@ -59,23 +50,21 @@ export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
|
||||
export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
|
||||
export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
|
||||
|
||||
#-J-Djdk.virtualThreadScheduler.parallelism=16
|
||||
|
||||
export OPTIONS="
|
||||
-A:jmx-remote -A:dev \
|
||||
-J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
|
||||
-J-Djdk.attach.allowAttachSelf \
|
||||
-J-Dpolyglot.engine.WarnInterpreterOnly=false \
|
||||
-J-Dlog4j2.configurationFile=log4j2-devenv.xml \
|
||||
-J-XX:+EnableDynamicAgentLoading \
|
||||
-J-XX:-OmitStackTraceInFastThrow \
|
||||
-J-XX:+UnlockDiagnosticVMOptions \
|
||||
-J-XX:+DebugNonSafepoints \
|
||||
-J-Djdk.tracePinnedThreads=full"
|
||||
|
||||
# Enable preview
|
||||
export OPTIONS="$OPTIONS -J--enable-preview"
|
||||
-J-Djdk.tracePinnedThreads=full \
|
||||
-J--enable-preview";
|
||||
|
||||
# Setup HEAP
|
||||
# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
|
||||
export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
|
||||
# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
|
||||
|
||||
# Increase virtual thread pool size
|
||||
@@ -88,7 +77,7 @@ export OPTIONS="$OPTIONS -J--enable-preview"
|
||||
# export OPTIONS="$OPTIONS -J-Xint"
|
||||
|
||||
# Setup GC
|
||||
# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
|
||||
export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
|
||||
|
||||
# Setup GC
|
||||
# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
|
||||
|
||||
@@ -6,90 +6,33 @@ export PENPOT_FLAGS="\
|
||||
$PENPOT_FLAGS \
|
||||
enable-prepl-server \
|
||||
enable-urepl-server \
|
||||
enable-nrepl-server \
|
||||
enable-webhooks \
|
||||
enable-backend-asserts \
|
||||
enable-audit-log \
|
||||
enable-transit-readable-response \
|
||||
enable-demo-users \
|
||||
enable-feature-fdata-pointer-map \
|
||||
enable-feature-fdata-objects-map \
|
||||
enable-fdata-storage-pointer-map \
|
||||
enable-fdata-storage-objets-map \
|
||||
disable-secure-session-cookies \
|
||||
enable-smtp \
|
||||
enable-access-tokens \
|
||||
disable-feature-components-v2 \
|
||||
enable-file-validation \
|
||||
enable-file-schema-validation \
|
||||
disable-soft-file-schema-validation \
|
||||
disable-soft-file-validation";
|
||||
enable-webhooks";
|
||||
|
||||
export OPTIONS="
|
||||
-A:jmx-remote -A:dev \
|
||||
-J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
|
||||
-J-Djdk.attach.allowAttachSelf \
|
||||
-J-Dpolyglot.engine.WarnInterpreterOnly=false \
|
||||
-J-Dlog4j2.configurationFile=log4j2.xml \
|
||||
-J-XX:+EnableDynamicAgentLoading \
|
||||
-J-XX:-OmitStackTraceInFastThrow \
|
||||
-J-XX:+UnlockDiagnosticVMOptions \
|
||||
-J-XX:+DebugNonSafepoints"
|
||||
|
||||
# Setup HEAP
|
||||
# export OPTIONS="$OPTIONS -J-Xms50m -J-Xmx1024m"
|
||||
# export OPTIONS="$OPTIONS -J-Xms1100m -J-Xmx1100m -J-XX:+AlwaysPreTouch"
|
||||
|
||||
# Increase virtual thread pool size
|
||||
# export OPTIONS="$OPTIONS -J-Djdk.virtualThreadScheduler.parallelism=16"
|
||||
|
||||
# Disable C2 Compiler
|
||||
# export OPTIONS="$OPTIONS -J-XX:TieredStopAtLevel=1"
|
||||
|
||||
# Disable all compilers
|
||||
# export OPTIONS="$OPTIONS -J-Xint"
|
||||
|
||||
# Setup GC
|
||||
# export OPTIONS="$OPTIONS -J-XX:+UseG1GC"
|
||||
|
||||
# Setup GC
|
||||
# export OPTIONS="$OPTIONS -J-XX:+UseZGC"
|
||||
|
||||
# Enable ImageMagick v7.x support
|
||||
# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
|
||||
|
||||
|
||||
# Initialize MINIO config
|
||||
mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin -q
|
||||
mc admin user add penpot-s3 penpot-devenv penpot-devenv -q
|
||||
mc admin user info penpot-s3 penpot-devenv |grep -F -q "readwrite"
|
||||
if [ "$?" = "1" ]; then
|
||||
mc admin policy attach penpot-s3 readwrite --user=penpot-devenv -q
|
||||
fi
|
||||
mc mb penpot-s3/penpot -p -q
|
||||
|
||||
export AWS_ACCESS_KEY_ID=penpot-devenv
|
||||
export AWS_SECRET_ACCESS_KEY=penpot-devenv
|
||||
export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
|
||||
export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
|
||||
export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
|
||||
set -ex
|
||||
|
||||
if [ "$1" = "--watch" ]; then
|
||||
trap "exit" INT TERM ERR
|
||||
trap "kill 0" EXIT
|
||||
|
||||
echo "Start Watch..."
|
||||
|
||||
clojure $OPTIONS -A:dev -M -m app.main &
|
||||
clojure -A:dev -M -m app.main &
|
||||
PID=$!
|
||||
|
||||
npx nodemon \
|
||||
--watch src \
|
||||
--watch ../common \
|
||||
--ext "clj" \
|
||||
--signal SIGKILL \
|
||||
--exec 'echo "(app.main/stop)\n\r(repl/refresh)\n\r(app.main/start)\n" | nc -N localhost 6062'
|
||||
|
||||
wait;
|
||||
--exec 'echo "(user/restart)" | nc -N localhost 6062'
|
||||
|
||||
kill -9 $PID
|
||||
else
|
||||
set -x
|
||||
clojure $OPTIONS -A:dev -M -m app.main;
|
||||
clojure -A:dev -M -m app.main
|
||||
fi
|
||||
|
||||
@@ -6,15 +6,14 @@
|
||||
|
||||
(ns app.auth
|
||||
(:require
|
||||
[app.config :as cf]
|
||||
[buddy.hashers :as hashers]
|
||||
[cuerdas.core :as str]))
|
||||
[promesa.exec :as px]))
|
||||
|
||||
(def default-params
|
||||
{:alg :argon2id
|
||||
:memory 32768 ;; 32 MiB
|
||||
:iterations 3
|
||||
:parallelism 2})
|
||||
:memory (* 32768 2)
|
||||
:iterations 5
|
||||
:parallelism (px/get-available-processors)})
|
||||
|
||||
(defn derive-password
|
||||
[password]
|
||||
@@ -28,16 +27,3 @@
|
||||
{:update false
|
||||
:valid false})))
|
||||
|
||||
(defn email-domain-in-whitelist?
|
||||
"Returns true if email's domain is in the given whitelist or if
|
||||
given whitelist is an empty string."
|
||||
([email]
|
||||
(let [domains (cf/get :registration-domain-whitelist)]
|
||||
(email-domain-in-whitelist? domains email)))
|
||||
([domains email]
|
||||
(if (or (nil? domains) (empty? domains))
|
||||
true
|
||||
(let [[_ candidate] (-> (str/lower email)
|
||||
(str/split #"@" 2))]
|
||||
(contains? domains candidate)))))
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
(ns app.auth.oidc
|
||||
"OIDC client implementation."
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.auth.oidc.providers :as-alias providers]
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
@@ -25,13 +24,11 @@
|
||||
[app.tokens :as tokens]
|
||||
[app.util.json :as json]
|
||||
[app.util.time :as dt]
|
||||
[buddy.sign.jwk :as jwk]
|
||||
[buddy.sign.jwt :as jwt]
|
||||
[clojure.set :as set]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.response :as-alias yrs]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HELPERS
|
||||
@@ -50,29 +47,36 @@
|
||||
|
||||
(defn- discover-oidc-config
|
||||
[cfg {:keys [base-uri] :as opts}]
|
||||
(let [uri (dm/str (u/join base-uri ".well-known/openid-configuration"))
|
||||
rsp (http/req! cfg {:method :get :uri uri} {:sync? true})]
|
||||
(if (= 200 (:status rsp))
|
||||
(let [data (-> rsp :body json/decode)
|
||||
(let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
|
||||
response (ex/try! (http/req! cfg
|
||||
{:method :get :uri (str discovery-uri)}
|
||||
{:sync? true}))]
|
||||
(cond
|
||||
(ex/exception? response)
|
||||
(do
|
||||
(l/warn :hint "unable to discover oidc configuration"
|
||||
:discover-uri (str discovery-uri)
|
||||
:cause response)
|
||||
nil)
|
||||
|
||||
(= 200 (:status response))
|
||||
(let [data (json/decode (:body response))
|
||||
token-uri (get data :token_endpoint)
|
||||
auth-uri (get data :authorization_endpoint)
|
||||
user-uri (get data :userinfo_endpoint)
|
||||
jwks-uri (get data :jwks_uri)]
|
||||
|
||||
user-uri (get data :userinfo_endpoint)]
|
||||
(l/debug :hint "oidc uris discovered"
|
||||
:token-uri token-uri
|
||||
:auth-uri auth-uri
|
||||
:user-uri user-uri
|
||||
:jwks-uri jwks-uri)
|
||||
|
||||
:user-uri user-uri)
|
||||
{:token-uri token-uri
|
||||
:auth-uri auth-uri
|
||||
:user-uri user-uri
|
||||
:jwks-uri jwks-uri})
|
||||
:user-uri user-uri})
|
||||
|
||||
:else
|
||||
(do
|
||||
(l/warn :hint "unable to discover OIDC configuration"
|
||||
:discover-uri uri
|
||||
:http-status (:status rsp))
|
||||
:uri (str discovery-uri)
|
||||
:response-status-code (:status response))
|
||||
nil))))
|
||||
|
||||
(defn- prepare-oidc-opts
|
||||
@@ -83,7 +87,6 @@
|
||||
:token-uri (cf/get :oidc-token-uri)
|
||||
:auth-uri (cf/get :oidc-auth-uri)
|
||||
:user-uri (cf/get :oidc-user-uri)
|
||||
:jwks-uri (cf/get :oidc-jwks-uri)
|
||||
:scopes (cf/get :oidc-scopes #{"openid" "profile" "email"})
|
||||
:roles-attr (cf/get :oidc-roles-attr)
|
||||
:roles (cf/get :oidc-roles)
|
||||
@@ -98,42 +101,8 @@
|
||||
(string? (:user-uri opts))
|
||||
(string? (:auth-uri opts)))
|
||||
opts
|
||||
(try
|
||||
(-> (discover-oidc-config cfg opts)
|
||||
(merge opts {:discover? true}))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unable to discover OIDC configuration"
|
||||
:cause cause)))))))
|
||||
|
||||
(defn- process-oidc-jwks
|
||||
[keys]
|
||||
(reduce (fn [result {:keys [kid] :as kdata}]
|
||||
(let [pkey (ex/try! (jwk/public-key kdata))]
|
||||
(if (ex/exception? pkey)
|
||||
(do
|
||||
(l/warn :hint "unable to create public key"
|
||||
:kid (:kid kdata)
|
||||
:cause pkey)
|
||||
result)
|
||||
(assoc result kid pkey))))
|
||||
{}
|
||||
keys))
|
||||
|
||||
(defn- fetch-oidc-jwks
|
||||
[cfg {:keys [jwks-uri]}]
|
||||
(when jwks-uri
|
||||
(try
|
||||
(let [{:keys [status body]} (http/req! cfg {:method :get :uri jwks-uri} {:sync? true})]
|
||||
(if (= 200 status)
|
||||
(-> body json/decode :keys process-oidc-jwks)
|
||||
(do
|
||||
(l/warn :hint "unable to retrieve JWKs (unexpected response status code)"
|
||||
:http-status status
|
||||
:http-body body)
|
||||
nil)))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unable to retrieve JWKs (unexpected exception)"
|
||||
:cause cause)))))
|
||||
(some-> (discover-oidc-config cfg opts)
|
||||
(merge opts {:discover? true}))))))
|
||||
|
||||
(defmethod ig/pre-init-spec ::providers/generic [_]
|
||||
(s/keys :req [::http/client]))
|
||||
@@ -142,7 +111,7 @@
|
||||
[_ cfg]
|
||||
(when (contains? cf/flags :login-with-oidc)
|
||||
(if-let [opts (prepare-oidc-opts cfg)]
|
||||
(let [jwks (fetch-oidc-jwks cfg opts)]
|
||||
(do
|
||||
(l/info :hint "provider initialized"
|
||||
:provider "oidc"
|
||||
:method (if (:discover? opts) "discover" "manual")
|
||||
@@ -153,9 +122,8 @@
|
||||
:user-uri (:user-uri opts)
|
||||
:token-uri (:token-uri opts)
|
||||
:roles-attr (:roles-attr opts)
|
||||
:roles (:roles opts)
|
||||
:keys (str/join "," (map str (keys jwks))))
|
||||
(assoc opts :jwks jwks))
|
||||
:roles (:roles opts))
|
||||
opts)
|
||||
(do
|
||||
(l/warn :hint "unable to initialize auth provider, missing configuration" :provider "oidc")
|
||||
nil))))
|
||||
@@ -196,7 +164,7 @@
|
||||
[cfg tdata props]
|
||||
(or (some-> props :github/email)
|
||||
(let [params {:uri "https://api.github.com/user/emails"
|
||||
:headers {"Authorization" (dm/str (:token/type tdata) " " (:token/access tdata))}
|
||||
:headers {"Authorization" (dm/str (:type tdata) " " (:token tdata))}
|
||||
:timeout 6000
|
||||
:method :get}
|
||||
|
||||
@@ -305,7 +273,7 @@
|
||||
{}
|
||||
props))
|
||||
|
||||
(defn fetch-access-token
|
||||
(defn retrieve-access-token
|
||||
[{:keys [provider] :as cfg} code]
|
||||
(let [params {:client_id (:client-id provider)
|
||||
:client_secret (:client-secret provider)
|
||||
@@ -329,9 +297,8 @@
|
||||
(l/trace :hint "access token response" :status status :body body)
|
||||
(if (= status 200)
|
||||
(let [data (json/decode body)]
|
||||
{:token/access (get data :access_token)
|
||||
:token/id (get data :id_token)
|
||||
:token/type (get data :token_type)})
|
||||
{:token (get data :access_token)
|
||||
:type (get data :token_type)})
|
||||
|
||||
(ex/raise :type :internal
|
||||
:code :unable-to-retrieve-token
|
||||
@@ -339,11 +306,12 @@
|
||||
:http-status status
|
||||
:http-body body)))))
|
||||
|
||||
(defn- process-user-info
|
||||
[provider tdata info]
|
||||
(defn- retrieve-user-info
|
||||
[{:keys [provider] :as cfg} tdata]
|
||||
(letfn [(get-email [props]
|
||||
;; Allow providers hook into this for custom email
|
||||
;; retrieval method.
|
||||
|
||||
(if-let [get-email-fn (:get-email-fn provider)]
|
||||
(get-email-fn tdata props)
|
||||
(let [attr-kw (cf/get :oidc-email-attr "email")
|
||||
@@ -353,54 +321,49 @@
|
||||
(get-name [props]
|
||||
(let [attr-kw (cf/get :oidc-name-attr "name")
|
||||
attr-ph (parse-attr-path provider attr-kw)]
|
||||
(get-in props attr-ph)))]
|
||||
(get-in props attr-ph)))
|
||||
|
||||
(let [props (qualify-props provider info)
|
||||
email (get-email props)]
|
||||
{:backend (:name provider)
|
||||
:fullname (or (get-name props) email)
|
||||
:email email
|
||||
:props props})))
|
||||
(process-response [response]
|
||||
(let [info (-> response :body json/decode)
|
||||
props (qualify-props provider info)
|
||||
email (get-email props)]
|
||||
{:backend (:name provider)
|
||||
:fullname (or (get-name props) email)
|
||||
:email email
|
||||
:props props}))]
|
||||
|
||||
(defn- fetch-user-info
|
||||
[{:keys [provider] :as cfg} tdata]
|
||||
(l/trace :hint "fetch user info"
|
||||
:uri (:user-uri provider)
|
||||
:token (obfuscate-string (:token/access tdata)))
|
||||
(l/trace :hint "request user info"
|
||||
:uri (:user-uri provider)
|
||||
:token (obfuscate-string (:token tdata))
|
||||
:token-type (:type tdata))
|
||||
|
||||
(let [params {:uri (:user-uri provider)
|
||||
:headers {"Authorization" (str (:token/type tdata) " " (:token/access tdata))}
|
||||
:timeout 6000
|
||||
:method :get}
|
||||
response (http/req! cfg params {:sync? true})]
|
||||
(let [request {:uri (:user-uri provider)
|
||||
:headers {"Authorization" (str (:type tdata) " " (:token tdata))}
|
||||
:timeout 6000
|
||||
:method :get}
|
||||
response (http/req! cfg request {:sync? true})]
|
||||
|
||||
(l/trace :hint "user info response"
|
||||
:status (:status response)
|
||||
:body (:body response))
|
||||
(l/trace :hint "user info response"
|
||||
:status (:status response)
|
||||
:body (:body response))
|
||||
|
||||
(when-not (s/int-in-range? 200 300 (:status response))
|
||||
(ex/raise :type :internal
|
||||
:code :unable-to-retrieve-user-info
|
||||
:hint "unable to retrieve user info"
|
||||
:http-status (:status response)
|
||||
:http-body (:body response)))
|
||||
(when-not (s/int-in-range? 200 300 (:status response))
|
||||
(ex/raise :type :internal
|
||||
:code :unable-to-retrieve-user-info
|
||||
:hint "unable to retrieve user info"
|
||||
:http-status (:status response)
|
||||
:http-body (:body response)))
|
||||
|
||||
(-> response :body json/decode)))
|
||||
(let [info (process-response response)]
|
||||
(l/trace :hint "authentication info" :info info)
|
||||
|
||||
(defn- get-user-info
|
||||
[{:keys [provider]} tdata]
|
||||
(try
|
||||
(when (:token/id tdata)
|
||||
(let [{:keys [kid alg] :as theader} (jwt/decode-header (:token/id tdata))]
|
||||
(when-let [key (if (str/starts-with? (name alg) "hs")
|
||||
(:client-secret provider)
|
||||
(get-in provider [:jwks kid]))]
|
||||
|
||||
(let [claims (jwt/unsign (:token/id tdata) key {:alg alg})]
|
||||
(dissoc claims :exp :iss :iat :sid :aud :sub)))))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unable to get user info from JWT token (unexpected exception)"
|
||||
:cause cause))))
|
||||
(when-not (s/valid? ::info info)
|
||||
(l/warn :hint "received incomplete profile info object (please set correct scopes)" :info info)
|
||||
(ex/raise :type :internal
|
||||
:code :incomplete-user-info
|
||||
:hint "inconmplete user info"
|
||||
:info info))
|
||||
info))))
|
||||
|
||||
(s/def ::backend ::us/not-empty-string)
|
||||
(s/def ::email ::us/not-empty-string)
|
||||
@@ -413,7 +376,7 @@
|
||||
::props]))
|
||||
|
||||
(defn get-info
|
||||
[{:keys [provider ::main/props] :as cfg} {:keys [params] :as request}]
|
||||
[{:keys [provider] :as cfg} {:keys [params] :as request}]
|
||||
(when-let [error (get params :error)]
|
||||
(ex/raise :type :internal
|
||||
:code :error-on-retrieving-code
|
||||
@@ -422,24 +385,9 @@
|
||||
|
||||
(let [state (get params :state)
|
||||
code (get params :code)
|
||||
state (tokens/verify props {:token state :iss :oauth})
|
||||
tdata (fetch-access-token cfg code)
|
||||
info (case (cf/get :oidc-user-info-source)
|
||||
:token (get-user-info cfg tdata)
|
||||
:userinfo (fetch-user-info cfg tdata)
|
||||
(or (get-user-info cfg tdata)
|
||||
(fetch-user-info cfg tdata)))
|
||||
|
||||
info (process-user-info provider tdata info)]
|
||||
|
||||
(l/trace :hint "user info" :info info)
|
||||
|
||||
(when-not (s/valid? ::info info)
|
||||
(l/warn :hint "received incomplete profile info object (please set correct scopes)" :info info)
|
||||
(ex/raise :type :internal
|
||||
:code :incomplete-user-info
|
||||
:hint "inconmplete user info"
|
||||
:info info))
|
||||
state (tokens/verify (::main/props cfg) {:token state :iss :oauth})
|
||||
token (retrieve-access-token cfg code)
|
||||
info (retrieve-user-info cfg token)]
|
||||
|
||||
;; If the provider is OIDC, we can proceed to check
|
||||
;; roles if they are defined.
|
||||
@@ -478,28 +426,14 @@
|
||||
|
||||
(defn- redirect-response
|
||||
[uri]
|
||||
{::rres/status 302
|
||||
::rres/headers {"location" (str uri)}})
|
||||
{::yrs/status 302
|
||||
::yrs/headers {"location" (str uri)}})
|
||||
|
||||
(defn- generate-error-redirect
|
||||
[_ cause]
|
||||
(let [data (if (ex/error? cause) (ex-data cause) nil)
|
||||
code (or (:code data) :unexpected)
|
||||
type (or (:type data) :internal)
|
||||
hint (or (:hint data)
|
||||
(if (ex/exception? cause)
|
||||
(ex-message cause)
|
||||
(str cause)))
|
||||
|
||||
params {:error "unable-to-auth"
|
||||
:hint hint
|
||||
:type type
|
||||
:code code}
|
||||
|
||||
uri (-> (u/uri (cf/get :public-uri))
|
||||
(assoc :path "/#/auth/login")
|
||||
(assoc :query (u/map->query-string params)))]
|
||||
|
||||
[_ error]
|
||||
(let [uri (-> (u/uri (cf/get :public-uri))
|
||||
(assoc :path "/#/auth/login")
|
||||
(assoc :query (u/map->query-string {:error "unable-to-auth" :hint (ex-message error)})))]
|
||||
(redirect-response uri)))
|
||||
|
||||
(defn- generate-redirect
|
||||
@@ -529,23 +463,19 @@
|
||||
(->> (redirect-response uri)
|
||||
(sxf request)))
|
||||
|
||||
(let [info (assoc info
|
||||
:iss :prepared-register
|
||||
:is-active true
|
||||
:exp (dt/in-future {:hours 48}))
|
||||
token (tokens/generate (::main/props cfg) info)
|
||||
params (d/without-nils
|
||||
{:token token
|
||||
:fullname (:fullname info)})
|
||||
uri (-> (u/uri (cf/get :public-uri))
|
||||
(assoc :path "/#/auth/register/validate")
|
||||
(assoc :query (u/map->query-string params)))]
|
||||
|
||||
(if (auth/email-domain-in-whitelist? (:email info))
|
||||
(let [info (assoc info
|
||||
:iss :prepared-register
|
||||
:is-active true
|
||||
:exp (dt/in-future {:hours 48}))
|
||||
token (tokens/generate (::main/props cfg) info)
|
||||
params (d/without-nils
|
||||
{:token token
|
||||
:fullname (:fullname info)})
|
||||
uri (-> (u/uri (cf/get :public-uri))
|
||||
(assoc :path "/#/auth/register/validate")
|
||||
(assoc :query (u/map->query-string params)))]
|
||||
|
||||
(redirect-response uri))
|
||||
(generate-error-redirect cfg "email-domain-not-allowed"))))
|
||||
|
||||
(redirect-response uri))))
|
||||
|
||||
(defn- auth-handler
|
||||
[cfg {:keys [params] :as request}]
|
||||
@@ -556,8 +486,8 @@
|
||||
:props props
|
||||
:exp (dt/in-future "4h")})
|
||||
uri (build-auth-uri cfg state)]
|
||||
{::rres/status 200
|
||||
::rres/body {:redirect-uri uri}}))
|
||||
{::yrs/status 200
|
||||
::yrs/body {:redirect-uri uri}}))
|
||||
|
||||
(defn- callback-handler
|
||||
[cfg request]
|
||||
@@ -566,7 +496,7 @@
|
||||
profile (get-profile cfg info)]
|
||||
(generate-redirect cfg request info profile))
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "error on oauth process" :cause cause)
|
||||
(l/error :hint "error on oauth process" :cause cause)
|
||||
(generate-error-redirect cfg cause))))
|
||||
|
||||
(def provider-lookup
|
||||
|
||||
@@ -146,13 +146,11 @@
|
||||
(s/def ::google-client-id ::us/string)
|
||||
(s/def ::google-client-secret ::us/string)
|
||||
(s/def ::oidc-client-id ::us/string)
|
||||
(s/def ::oidc-user-info-source ::us/keyword)
|
||||
(s/def ::oidc-client-secret ::us/string)
|
||||
(s/def ::oidc-base-uri ::us/string)
|
||||
(s/def ::oidc-token-uri ::us/string)
|
||||
(s/def ::oidc-auth-uri ::us/string)
|
||||
(s/def ::oidc-user-uri ::us/string)
|
||||
(s/def ::oidc-jwks-uri ::us/string)
|
||||
(s/def ::oidc-scopes ::us/set-of-strings)
|
||||
(s/def ::oidc-roles ::us/set-of-strings)
|
||||
(s/def ::oidc-roles-attr ::us/string)
|
||||
@@ -203,7 +201,6 @@
|
||||
(s/def ::storage-assets-s3-bucket ::us/string)
|
||||
(s/def ::storage-assets-s3-region ::us/keyword)
|
||||
(s/def ::storage-assets-s3-endpoint ::us/string)
|
||||
(s/def ::storage-assets-s3-io-threads ::us/integer)
|
||||
(s/def ::telemetry-uri ::us/string)
|
||||
(s/def ::telemetry-with-taiga ::us/boolean)
|
||||
(s/def ::tenant ::us/string)
|
||||
@@ -244,12 +241,10 @@
|
||||
::google-client-secret
|
||||
::oidc-client-id
|
||||
::oidc-client-secret
|
||||
::oidc-user-info-source
|
||||
::oidc-base-uri
|
||||
::oidc-token-uri
|
||||
::oidc-auth-uri
|
||||
::oidc-user-uri
|
||||
::oidc-jwks-uri
|
||||
::oidc-scopes
|
||||
::oidc-roles-attr
|
||||
::oidc-email-attr
|
||||
@@ -295,7 +290,6 @@
|
||||
::redis-uri
|
||||
::registration-domain-whitelist
|
||||
::rpc-rlimit-config
|
||||
::rpc-climit-config
|
||||
|
||||
::semaphore-process-font
|
||||
::semaphore-process-image
|
||||
@@ -321,7 +315,6 @@
|
||||
::storage-assets-s3-bucket
|
||||
::storage-assets-s3-region
|
||||
::storage-assets-s3-endpoint
|
||||
::storage-assets-s3-io-threads
|
||||
::telemetry-enabled
|
||||
::telemetry-uri
|
||||
::telemetry-referer
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.db
|
||||
(:refer-clojure :exclude [get run!])
|
||||
(:refer-clojure :exclude [get])
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
@@ -97,7 +97,7 @@
|
||||
:with-credentials (and (contains? cfg ::username)
|
||||
(contains? cfg ::password))
|
||||
:min-size (::min-size cfg)
|
||||
:max-size (::max-size cfg))
|
||||
:max-size (::max-size cfg))
|
||||
(create-pool cfg)))
|
||||
|
||||
(defmethod ig/halt-key! ::pool
|
||||
@@ -145,10 +145,6 @@
|
||||
[v]
|
||||
(instance? javax.sql.DataSource v))
|
||||
|
||||
(defn connection?
|
||||
[conn]
|
||||
(instance? Connection conn))
|
||||
|
||||
(s/def ::conn some?)
|
||||
(s/def ::nilable-pool (s/nilable ::pool))
|
||||
(s/def ::pool pool?)
|
||||
@@ -222,91 +218,52 @@
|
||||
|
||||
(defmacro with-atomic
|
||||
[& args]
|
||||
(if (symbol? (first args))
|
||||
(let [cfgs (first args)
|
||||
body (rest args)]
|
||||
`(jdbc/with-transaction [conn# (::pool ~cfgs)]
|
||||
(let [~cfgs (assoc ~cfgs ::conn conn#)]
|
||||
~@body)))
|
||||
`(jdbc/with-transaction ~@args)))
|
||||
`(jdbc/with-transaction ~@args))
|
||||
|
||||
(defn open
|
||||
[system-or-pool]
|
||||
(if (pool? system-or-pool)
|
||||
(jdbc/get-connection system-or-pool)
|
||||
(if (map? system-or-pool)
|
||||
(open (::pool system-or-pool))
|
||||
(ex/raise :type :internal
|
||||
:code :unable-resolve-pool))))
|
||||
|
||||
(defn get-connection
|
||||
[cfg-or-conn]
|
||||
(if (connection? cfg-or-conn)
|
||||
cfg-or-conn
|
||||
(if (map? cfg-or-conn)
|
||||
(get-connection (::conn cfg-or-conn))
|
||||
(ex/raise :type :internal
|
||||
:code :unable-resolve-connection
|
||||
:hint "expected conn or system map"))))
|
||||
|
||||
(defn connection-map?
|
||||
"Check if the provided value is a map like data structure that
|
||||
contains a database connection."
|
||||
[o]
|
||||
(and (map? o) (connection? (::conn o))))
|
||||
|
||||
(defn- get-connectable
|
||||
[o]
|
||||
(cond
|
||||
(connection? o) o
|
||||
(pool? o) o
|
||||
(map? o) (get-connectable (or (::conn o) (::pool o)))
|
||||
:else (ex/raise :type :internal
|
||||
:code :unable-resolve-connectable
|
||||
:hint "expected conn, pool or system")))
|
||||
[pool]
|
||||
(jdbc/get-connection pool))
|
||||
|
||||
(def ^:private default-opts
|
||||
{:builder-fn sql/as-kebab-maps})
|
||||
|
||||
(defn exec!
|
||||
([ds sv]
|
||||
(-> (get-connectable ds)
|
||||
(jdbc/execute! sv default-opts)))
|
||||
(jdbc/execute! ds sv default-opts))
|
||||
([ds sv opts]
|
||||
(-> (get-connectable ds)
|
||||
(jdbc/execute! sv (into default-opts (sql/adapt-opts opts))))))
|
||||
(jdbc/execute! ds sv (merge default-opts opts))))
|
||||
|
||||
(defn exec-one!
|
||||
([ds sv]
|
||||
(-> (get-connectable ds)
|
||||
(jdbc/execute-one! sv default-opts)))
|
||||
(jdbc/execute-one! ds sv default-opts))
|
||||
([ds sv opts]
|
||||
(-> (get-connectable ds)
|
||||
(jdbc/execute-one! sv (into default-opts (sql/adapt-opts opts))))))
|
||||
(jdbc/execute-one! ds sv
|
||||
(-> (merge default-opts opts)
|
||||
(assoc :return-keys (::return-keys? opts false))))))
|
||||
|
||||
(defn insert!
|
||||
[ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
|
||||
(-> (get-connectable ds)
|
||||
(exec-one! (sql/insert table params opts)
|
||||
(assoc opts ::return-keys? return-keys?))))
|
||||
[ds table params & {:as opts}]
|
||||
(exec-one! ds
|
||||
(sql/insert table params opts)
|
||||
(merge {::return-keys? true} opts)))
|
||||
|
||||
(defn insert-multi!
|
||||
[ds table cols rows & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
|
||||
(-> (get-connectable ds)
|
||||
(exec! (sql/insert-multi table cols rows opts)
|
||||
(assoc opts ::return-keys? return-keys?))))
|
||||
[ds table cols rows & {:as opts}]
|
||||
(exec! ds
|
||||
(sql/insert-multi table cols rows opts)
|
||||
(merge {::return-keys? true} opts)))
|
||||
|
||||
(defn update!
|
||||
[ds table params where & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
|
||||
(-> (get-connectable ds)
|
||||
(exec-one! (sql/update table params where opts)
|
||||
(assoc opts ::return-keys? return-keys?))))
|
||||
[ds table params where & {:as opts}]
|
||||
(exec-one! ds
|
||||
(sql/update table params where opts)
|
||||
(merge {::return-keys? true} opts)))
|
||||
|
||||
(defn delete!
|
||||
[ds table params & {:as opts :keys [::return-keys?] :or {return-keys? true}}]
|
||||
(-> (get-connectable ds)
|
||||
(exec-one! (sql/delete table params opts)
|
||||
(assoc opts ::return-keys? return-keys?))))
|
||||
[ds table params & {:as opts}]
|
||||
(exec-one! ds
|
||||
(sql/delete table params opts)
|
||||
(merge {::return-keys? true} opts)))
|
||||
|
||||
(defn is-row-deleted?
|
||||
[{:keys [deleted-at]}]
|
||||
@@ -336,11 +293,6 @@
|
||||
:hint "database object not found"))
|
||||
row))
|
||||
|
||||
(defn plan
|
||||
[ds sql]
|
||||
(-> (get-connectable ds)
|
||||
(jdbc/plan sql sql/default-opts)))
|
||||
|
||||
(defn get-by-id
|
||||
[ds table id & {:as opts}]
|
||||
(get ds table {:id id} opts))
|
||||
@@ -409,6 +361,10 @@
|
||||
[data]
|
||||
(org.postgresql.util.PGInterval. ^String data))
|
||||
|
||||
(defn connection?
|
||||
[conn]
|
||||
(instance? Connection conn))
|
||||
|
||||
(defn savepoint
|
||||
([^Connection conn]
|
||||
(.setSavepoint conn))
|
||||
@@ -416,68 +372,14 @@
|
||||
(.setSavepoint conn (name label))))
|
||||
|
||||
(defn release!
|
||||
[^Connection conn ^Savepoint sp]
|
||||
[^Connection conn ^Savepoint sp ]
|
||||
(.releaseSavepoint conn sp))
|
||||
|
||||
(defn rollback!
|
||||
([conn]
|
||||
(let [^Connection conn (get-connection conn)]
|
||||
(l/trc :hint "explicit rollback requested")
|
||||
(.rollback conn)))
|
||||
([conn ^Savepoint sp]
|
||||
(let [^Connection conn (get-connection conn)]
|
||||
(l/trc :hint "explicit rollback requested")
|
||||
(.rollback conn sp))))
|
||||
|
||||
(defn tx-run!
|
||||
[system f & params]
|
||||
(cond
|
||||
(connection? system)
|
||||
(tx-run! {::conn system} f)
|
||||
|
||||
(pool? system)
|
||||
(tx-run! {::pool system} f)
|
||||
|
||||
(::conn system)
|
||||
(let [conn (::conn system)
|
||||
sp (savepoint conn)]
|
||||
(try
|
||||
(let [result (apply f system params)]
|
||||
(release! conn sp)
|
||||
result)
|
||||
(catch Throwable cause
|
||||
(rollback! conn sp)
|
||||
(throw cause))))
|
||||
|
||||
(::pool system)
|
||||
(with-atomic [conn (::pool system)]
|
||||
(let [system (assoc system ::conn conn)
|
||||
result (apply f system params)]
|
||||
(when (::rollback system)
|
||||
(rollback! conn))
|
||||
result))
|
||||
|
||||
:else
|
||||
(throw (IllegalArgumentException. "invalid arguments"))))
|
||||
|
||||
(defn run!
|
||||
[system f & params]
|
||||
(cond
|
||||
(connection? system)
|
||||
(run! {::conn system} f)
|
||||
|
||||
(pool? system)
|
||||
(run! {::pool system} f)
|
||||
|
||||
(::conn system)
|
||||
(apply f system params)
|
||||
|
||||
(::pool system)
|
||||
(with-open [^Connection conn (open (::pool system))]
|
||||
(apply f (assoc system ::conn conn) params))
|
||||
|
||||
:else
|
||||
(throw (IllegalArgumentException. "invalid arguments"))))
|
||||
([^Connection conn]
|
||||
(.rollback conn))
|
||||
([^Connection conn ^Savepoint sp]
|
||||
(.rollback conn sp)))
|
||||
|
||||
(defn interval
|
||||
[o]
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
(:refer-clojure :exclude [update])
|
||||
(:require
|
||||
[app.db :as-alias db]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as str]
|
||||
[next.jdbc.optional :as jdbc-opt]
|
||||
[next.jdbc.sql.builder :as sql]))
|
||||
@@ -20,14 +19,6 @@
|
||||
{:table-fn snake-case
|
||||
:column-fn snake-case})
|
||||
|
||||
(def params-mapping
|
||||
{::db/return-keys? :return-keys
|
||||
::db/columns :columns})
|
||||
|
||||
(defn adapt-opts
|
||||
[opts]
|
||||
(set/rename-keys opts params-mapping))
|
||||
|
||||
(defn as-kebab-maps
|
||||
[rs opts]
|
||||
(jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
|
||||
@@ -38,7 +29,7 @@
|
||||
([table key-map opts]
|
||||
(let [opts (merge default-opts opts)
|
||||
opts (cond-> opts
|
||||
(::db/on-conflict-do-nothing? opts)
|
||||
(:on-conflict-do-nothing opts)
|
||||
(assoc :suffix "ON CONFLICT DO NOTHING"))]
|
||||
(sql/for-insert table key-map opts))))
|
||||
|
||||
@@ -53,7 +44,6 @@
|
||||
([table where-params opts]
|
||||
(let [opts (merge default-opts opts)
|
||||
opts (cond-> opts
|
||||
(::db/columns opts) (assoc :columns (::db/columns opts))
|
||||
(::db/for-update? opts) (assoc :suffix "FOR UPDATE")
|
||||
(::db/for-share? opts) (assoc :suffix "FOR KEY SHARE")
|
||||
(:for-update opts) (assoc :suffix "FOR UPDATE")
|
||||
@@ -64,13 +54,7 @@
|
||||
([table key-map where-params]
|
||||
(update table key-map where-params nil))
|
||||
([table key-map where-params opts]
|
||||
(let [opts (into default-opts opts)
|
||||
opts (if-let [columns (::db/columns opts)]
|
||||
(let [columns (if (seq columns)
|
||||
(sql/as-cols columns opts)
|
||||
"*")]
|
||||
(assoc opts :suffix (str "RETURNING " columns)))
|
||||
opts)]
|
||||
(let [opts (merge default-opts opts)]
|
||||
(sql/for-update table key-map where-params opts))))
|
||||
|
||||
(defn delete
|
||||
|
||||
@@ -305,18 +305,12 @@
|
||||
(fn [params]
|
||||
(when (contains? cf/flags :smtp)
|
||||
(let [session (create-smtp-session cfg)]
|
||||
(with-open [transport (.getTransport session (if (::ssl cfg) "smtps" "smtp"))]
|
||||
(with-open [transport (.getTransport session (if (:ssl cfg) "smtps" "smtp"))]
|
||||
(.connect ^Transport transport
|
||||
^String (::username cfg)
|
||||
^String (::password cfg))
|
||||
|
||||
(let [^MimeMessage message (create-smtp-message cfg session params)]
|
||||
(l/dbg :hint "sendmail"
|
||||
:id (:id params)
|
||||
:to (:to params)
|
||||
:subject (str/trim (:subject params))
|
||||
:body (str/join "," (map :type (:body params))))
|
||||
|
||||
(.sendMessage ^Transport transport
|
||||
^MimeMessage message
|
||||
(.getAllRecipients message))))))
|
||||
@@ -347,7 +341,7 @@
|
||||
(map :content)
|
||||
first)))
|
||||
(println "******** end email" (:id email) "**********"))]
|
||||
(l/raw! :info out)))
|
||||
(l/info ::l/raw out)))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; EMAIL FACTORIES
|
||||
|
||||
@@ -1,927 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.features.components-v2
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.files.changes :as cp]
|
||||
[app.common.files.changes-builder :as fcb]
|
||||
[app.common.files.helpers :as cfh]
|
||||
[app.common.files.libraries-helpers :as cflh]
|
||||
[app.common.files.migrations :as fmg]
|
||||
[app.common.files.shapes-helpers :as cfsh]
|
||||
[app.common.files.validate :as cfv]
|
||||
[app.common.geom.point :as gpt]
|
||||
[app.common.geom.rect :as grc]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.logging :as l]
|
||||
[app.common.svg :as csvg]
|
||||
[app.common.svg.shapes-builder :as sbuilder]
|
||||
[app.common.types.component :as ctk]
|
||||
[app.common.types.components-list :as ctkl]
|
||||
[app.common.types.container :as ctn]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.types.pages-list :as ctpl]
|
||||
[app.common.types.shape :as cts]
|
||||
[app.common.types.shape-tree :as ctst]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as fdata]
|
||||
[app.http.sse :as sse]
|
||||
[app.media :as media]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.files-snapshot :as fsnap]
|
||||
[app.rpc.commands.media :as cmd.media]
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.time :as dt]
|
||||
[buddy.core.codecs :as bc]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.io :as io]
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.semaphore :as ps]
|
||||
[promesa.util :as pu]))
|
||||
|
||||
(def ^:dynamic *system* nil)
|
||||
(def ^:dynamic *stats* nil)
|
||||
(def ^:dynamic *file-stats* nil)
|
||||
(def ^:dynamic *team-stats* nil)
|
||||
(def ^:dynamic *semaphore* nil)
|
||||
(def ^:dynamic *skip-on-error* true)
|
||||
|
||||
(def grid-gap 50)
|
||||
(def frame-gap 200)
|
||||
(def max-group-size 50)
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; FILE PREPARATION BEFORE MIGRATION
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- prepare-file-data
|
||||
"Apply some specific migrations or fixes to things that are allowed in v1 but not in v2,
|
||||
or that are the result of old bugs."
|
||||
[file-data libraries]
|
||||
(let [detached-ids (volatile! #{})
|
||||
|
||||
detach-shape
|
||||
(fn [container shape]
|
||||
;; Detach a shape. If it's inside a component, add it to detached-ids, for further use.
|
||||
(let [is-component? (let [root-shape (ctst/get-shape container (:id container))]
|
||||
(and (some? root-shape) (nil? (:parent-id root-shape))))]
|
||||
(when is-component?
|
||||
(vswap! detached-ids conj (:id shape)))
|
||||
(ctk/detach-shape shape)))
|
||||
|
||||
fix-orphan-shapes
|
||||
(fn [file-data]
|
||||
;; Find shapes that are not listed in their parent's children list.
|
||||
;; Remove them, and also their children
|
||||
(letfn [(fix-container [container]
|
||||
(reduce fix-shape container (ctn/shapes-seq container)))
|
||||
|
||||
(fix-shape
|
||||
[container shape]
|
||||
(if-not (or (= (:id shape) uuid/zero)
|
||||
(nil? (:parent-id shape)))
|
||||
(let [parent (ctst/get-shape container (:parent-id shape))
|
||||
exists? (d/index-of (:shapes parent) (:id shape))]
|
||||
(if (nil? exists?)
|
||||
(let [ids (cfh/get-children-ids-with-self (:objects container) (:id shape))]
|
||||
(update container :objects #(reduce dissoc % ids)))
|
||||
container))
|
||||
container))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
remove-nested-roots
|
||||
(fn [file-data]
|
||||
;; Remove :component-root in head shapes that are nested.
|
||||
(letfn [(fix-container [container]
|
||||
(update container :objects update-vals (partial fix-shape container)))
|
||||
|
||||
(fix-shape [container shape]
|
||||
(let [parent (ctst/get-shape container (:parent-id shape))]
|
||||
(if (and (ctk/instance-root? shape)
|
||||
(ctn/in-any-component? (:objects container) parent))
|
||||
(dissoc shape :component-root)
|
||||
shape)))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
add-not-nested-roots
|
||||
(fn [file-data]
|
||||
;; Add :component-root in head shapes that are not nested.
|
||||
(letfn [(fix-container [container]
|
||||
(update container :objects update-vals (partial fix-shape container)))
|
||||
|
||||
(fix-shape [container shape]
|
||||
(let [parent (ctst/get-shape container (:parent-id shape))]
|
||||
(if (and (ctk/subinstance-head? shape)
|
||||
(not (ctn/in-any-component? (:objects container) parent)))
|
||||
(assoc shape :component-root true)
|
||||
shape)))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
fix-orphan-copies
|
||||
(fn [file-data]
|
||||
;; Detach shapes that were inside a copy (have :shape-ref) but now they aren't.
|
||||
(letfn [(fix-container [container]
|
||||
(update container :objects update-vals (partial fix-shape container)))
|
||||
|
||||
(fix-shape [container shape]
|
||||
(let [parent (ctst/get-shape container (:parent-id shape))]
|
||||
(if (and (ctk/in-component-copy? shape)
|
||||
(not (ctk/instance-head? shape))
|
||||
(not (ctk/in-component-copy? parent)))
|
||||
(detach-shape container shape)
|
||||
shape)))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
remap-refs
|
||||
(fn [file-data]
|
||||
;; Remap shape-refs so that they point to the near main.
|
||||
;; At the same time, if there are any dangling ref, detach the shape and its children.
|
||||
(letfn [(fix-container [container]
|
||||
(reduce fix-shape container (ctn/shapes-seq container)))
|
||||
|
||||
(fix-shape [container shape]
|
||||
(if (ctk/in-component-copy? shape)
|
||||
;; First look for the direct shape.
|
||||
(let [root (ctn/get-component-shape (:objects container) shape)
|
||||
libraries (assoc-in libraries [(:id file-data) :data] file-data)
|
||||
library (get libraries (:component-file root))
|
||||
component (ctkl/get-component (:data library) (:component-id root) true)
|
||||
direct-shape (ctf/get-component-shape (:data library) component (:shape-ref shape))]
|
||||
(if (some? direct-shape)
|
||||
;; If it exists, there is nothing else to do.
|
||||
container
|
||||
;; If not found, find the near shape.
|
||||
(let [near-shape (d/seek #(= (:shape-ref %) (:shape-ref shape))
|
||||
(ctf/get-component-shapes (:data library) component))]
|
||||
(if (some? near-shape)
|
||||
;; If found, update the ref to point to the near shape.
|
||||
(ctn/update-shape container (:id shape) #(assoc % :shape-ref (:id near-shape)))
|
||||
;; If not found, it may be a fostered component. Try to locate a direct shape
|
||||
;; in the head component.
|
||||
(let [head (ctn/get-head-shape (:objects container) shape)
|
||||
library-2 (get libraries (:component-file head))
|
||||
component-2 (ctkl/get-component (:data library-2) (:component-id head) true)
|
||||
direct-shape-2 (ctf/get-component-shape (:data library-2) component-2 (:shape-ref shape))]
|
||||
(if (some? direct-shape-2)
|
||||
;; If it exists, there is nothing else to do.
|
||||
container
|
||||
;; If not found, detach shape and all children (stopping if a nested instance is reached)
|
||||
(let [children (ctn/get-children-in-instance (:objects container) (:id shape))]
|
||||
(reduce #(ctn/update-shape %1 (:id %2) (partial detach-shape %1))
|
||||
container
|
||||
children))))))))
|
||||
container))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
fix-copies-of-detached
|
||||
(fn [file-data]
|
||||
;; Find any copy that is referencing a detached shape inside a component, and
|
||||
;; undo the nested copy, converting it into a direct copy.
|
||||
(letfn [(fix-container [container]
|
||||
(update container :objects update-vals fix-shape))
|
||||
|
||||
(fix-shape [shape]
|
||||
(cond-> shape
|
||||
(@detached-ids (:shape-ref shape))
|
||||
(dissoc shape
|
||||
:component-id
|
||||
:component-file
|
||||
:component-root)))]
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
transform-to-frames
|
||||
(fn [file-data]
|
||||
;; Transform component and copy heads to frames, and set the
|
||||
;; frame-id of its childrens
|
||||
(letfn [(fix-container
|
||||
[container]
|
||||
(update container :objects update-vals fix-shape))
|
||||
|
||||
(fix-shape
|
||||
[shape]
|
||||
(if (or (nil? (:parent-id shape)) (ctk/instance-head? shape))
|
||||
(assoc shape
|
||||
:type :frame ; Old groups must be converted
|
||||
:fills (or (:fills shape) []) ; to frames and conform to spec
|
||||
:hide-in-viewer (or (:hide-in-viewer shape) true)
|
||||
:rx (or (:rx shape) 0)
|
||||
:ry (or (:ry shape) 0))
|
||||
shape))]
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
remap-frame-ids
|
||||
(fn [file-data]
|
||||
;; Remap the frame-ids of the primary childs of the head instances
|
||||
;; to point to the head instance.
|
||||
(letfn [(fix-container
|
||||
[container]
|
||||
(update container :objects update-vals (partial fix-shape container)))
|
||||
|
||||
(fix-shape
|
||||
[container shape]
|
||||
(let [parent (ctst/get-shape container (:parent-id shape))]
|
||||
(if (ctk/instance-head? parent)
|
||||
(assoc shape :frame-id (:id parent))
|
||||
shape)))]
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
fix-frame-ids
|
||||
(fn [file-data]
|
||||
;; Ensure that frame-id of all shapes point to the parent or to the frame-id
|
||||
;; of the parent, and that the destination is indeed a frame.
|
||||
(letfn [(fix-container [container]
|
||||
(update container :objects #(cfh/reduce-objects % fix-shape %)))
|
||||
|
||||
(fix-shape [objects shape]
|
||||
(let [parent (when (:parent-id shape)
|
||||
(get objects (:parent-id shape)))
|
||||
error? (when (some? parent)
|
||||
(if (= (:type parent) :frame)
|
||||
(not= (:frame-id shape) (:id parent))
|
||||
(not= (:frame-id shape) (:frame-id parent))))]
|
||||
(if error?
|
||||
(let [nearest-frame (cfh/get-frame objects (:parent-id shape))
|
||||
frame-id (or (:id nearest-frame) uuid/zero)]
|
||||
(update objects (:id shape) assoc :frame-id frame-id))
|
||||
objects)))]
|
||||
|
||||
(-> file-data
|
||||
(update :pages-index update-vals fix-container)
|
||||
(update :components update-vals fix-container))))
|
||||
|
||||
fix-component-nil-objects
|
||||
(fn [file-data]
|
||||
;; Ensure that objects of all components is not null
|
||||
(letfn [(fix-component [component]
|
||||
(if (and (contains? component :objects) (nil? (:objects component)))
|
||||
(if (:deleted component)
|
||||
(assoc component :objects {})
|
||||
(dissoc component :objects))
|
||||
component))]
|
||||
(-> file-data
|
||||
(update :components update-vals fix-component))))]
|
||||
|
||||
(-> file-data
|
||||
(fix-orphan-shapes)
|
||||
(remove-nested-roots)
|
||||
(add-not-nested-roots)
|
||||
(fix-orphan-copies)
|
||||
(remap-refs)
|
||||
(fix-copies-of-detached)
|
||||
(transform-to-frames)
|
||||
(remap-frame-ids)
|
||||
(fix-frame-ids)
|
||||
(fix-component-nil-objects))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; COMPONENTS MIGRATION
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- get-asset-groups
|
||||
[assets generic-name]
|
||||
(let [; Group by first element of the path.
|
||||
groups (d/group-by #(first (cfh/split-path (:path %))) assets)
|
||||
|
||||
; Split large groups in chunks of max-group-size elements
|
||||
groups (loop [groups (seq groups)
|
||||
result {}]
|
||||
(if (empty? groups)
|
||||
result
|
||||
(let [[group-name assets] (first groups)
|
||||
group-name (if (or (nil? group-name) (str/empty? group-name))
|
||||
generic-name
|
||||
group-name)]
|
||||
(if (<= (count assets) max-group-size)
|
||||
(recur (next groups)
|
||||
(assoc result group-name assets))
|
||||
(let [splits (-> (partition-all max-group-size assets)
|
||||
(d/enumerate))]
|
||||
(recur (next groups)
|
||||
(reduce (fn [result [index split]]
|
||||
(let [split-name (str group-name " " (inc index))]
|
||||
(assoc result split-name split)))
|
||||
result
|
||||
splits)))))))
|
||||
|
||||
; Sort assets in each group by path
|
||||
groups (update-vals groups (fn [assets]
|
||||
(sort-by (fn [{:keys [path name]}]
|
||||
(str/lower (cfh/merge-path-item path name)))
|
||||
assets)))
|
||||
|
||||
; Sort groups by name
|
||||
groups (into (sorted-map) groups)]
|
||||
groups))
|
||||
|
||||
(defn- create-frame
|
||||
[name position width height]
|
||||
(cts/setup-shape
|
||||
{:type :frame
|
||||
:x (:x position)
|
||||
:y (:y position)
|
||||
:width (+ width (* 2 grid-gap))
|
||||
:height (+ height (* 2 grid-gap))
|
||||
:name name
|
||||
:frame-id uuid/zero
|
||||
:parent-id uuid/zero}))
|
||||
|
||||
(defn- migrate-components
|
||||
"If there is any component in the file library, add a new 'Library
|
||||
backup', generate main instances for all components there and remove
|
||||
shapes from library components. Mark the file with
|
||||
the :components-v2 option."
|
||||
[file-data libraries]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :components})
|
||||
(let [components (ctkl/components-seq file-data)]
|
||||
(if (empty? components)
|
||||
(assoc-in file-data [:options :components-v2] true)
|
||||
(let [[file-data page-id start-pos]
|
||||
(ctf/get-or-add-library-page file-data frame-gap)
|
||||
|
||||
migrate-component-shape
|
||||
(fn [shape delta component-file component-id frame-id]
|
||||
(cond-> shape
|
||||
(nil? (:parent-id shape))
|
||||
(assoc :parent-id frame-id
|
||||
:main-instance true
|
||||
:component-root true
|
||||
:component-file component-file
|
||||
:component-id component-id)
|
||||
|
||||
(nil? (:frame-id shape))
|
||||
(assoc :frame-id frame-id)
|
||||
|
||||
:always
|
||||
(gsh/move delta)))
|
||||
|
||||
add-main-instance
|
||||
(fn [file-data component frame-id position]
|
||||
(let [shapes (cfh/get-children-with-self (:objects component)
|
||||
(:id component))
|
||||
|
||||
root-shape (first shapes)
|
||||
orig-pos (gpt/point (:x root-shape) (:y root-shape))
|
||||
delta (gpt/subtract position orig-pos)
|
||||
|
||||
xf-shape (map #(migrate-component-shape %
|
||||
delta
|
||||
(:id file-data)
|
||||
(:id component)
|
||||
frame-id))
|
||||
new-shapes
|
||||
(into [] xf-shape shapes)
|
||||
|
||||
find-frame-id ; if its parent is a frame, the frame-id should be the parent-id
|
||||
(fn [page shape]
|
||||
(let [parent (ctst/get-shape page (:parent-id shape))]
|
||||
(if (= :frame (:type parent))
|
||||
(:id parent)
|
||||
(:frame-id parent))))
|
||||
|
||||
add-shapes
|
||||
(fn [page]
|
||||
(reduce (fn [page shape]
|
||||
(ctst/add-shape (:id shape)
|
||||
shape
|
||||
page
|
||||
(find-frame-id page shape)
|
||||
(:parent-id shape)
|
||||
nil ; <- As shapes are ordered, we can safely add each
|
||||
true)) ; one at the end of the parent's children list.
|
||||
page
|
||||
new-shapes))
|
||||
|
||||
update-component
|
||||
(fn [component]
|
||||
(-> component
|
||||
(assoc :main-instance-id (:id root-shape)
|
||||
:main-instance-page page-id)
|
||||
(dissoc :objects)))]
|
||||
|
||||
(-> file-data
|
||||
(ctpl/update-page page-id add-shapes)
|
||||
(ctkl/update-component (:id component) update-component))))
|
||||
|
||||
add-instance-grid
|
||||
(fn [fdata frame-id grid assets]
|
||||
(reduce (fn [result [component position]]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :components
|
||||
:name (:name component)})
|
||||
(add-main-instance result component frame-id (gpt/add position
|
||||
(gpt/point grid-gap grid-gap))))
|
||||
fdata
|
||||
(d/zip assets grid)))
|
||||
|
||||
add-instance-grids
|
||||
(fn [fdata]
|
||||
(let [components (ctkl/components-seq fdata)
|
||||
groups (get-asset-groups components "Components")]
|
||||
(loop [groups (seq groups)
|
||||
fdata fdata
|
||||
position start-pos]
|
||||
(if (empty? groups)
|
||||
fdata
|
||||
(let [[group-name assets] (first groups)
|
||||
grid (ctst/generate-shape-grid
|
||||
(map (partial ctf/get-component-root fdata) assets)
|
||||
position
|
||||
grid-gap)
|
||||
{:keys [width height]} (meta grid)
|
||||
frame (create-frame group-name position width height)
|
||||
fdata (ctpl/update-page fdata
|
||||
page-id
|
||||
#(ctst/add-shape (:id frame)
|
||||
frame
|
||||
%
|
||||
(:id frame)
|
||||
(:id frame)
|
||||
nil
|
||||
true))]
|
||||
(recur (next groups)
|
||||
(add-instance-grid fdata (:id frame) grid assets)
|
||||
(gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))]
|
||||
|
||||
(let [total (count components)]
|
||||
(some-> *stats* (swap! update :processed/components (fnil + 0) total))
|
||||
(some-> *team-stats* (swap! update :processed/components (fnil + 0) total))
|
||||
(some-> *file-stats* (swap! assoc :processed/components total)))
|
||||
|
||||
(-> file-data
|
||||
(prepare-file-data libraries)
|
||||
(add-instance-grids))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; GRAPHICS MIGRATION
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- create-shapes-for-bitmap
|
||||
"Convert a media object that contains a bitmap image into shapes,
|
||||
one shape of type :image and one group that contains it."
|
||||
[{:keys [name width height id mtype]} frame-id position]
|
||||
(let [frame-shape (cts/setup-shape
|
||||
{:type :frame
|
||||
:x (:x position)
|
||||
:y (:y position)
|
||||
:width width
|
||||
:height height
|
||||
:name name
|
||||
:frame-id frame-id
|
||||
:parent-id frame-id})
|
||||
|
||||
img-shape (cts/setup-shape
|
||||
{:type :image
|
||||
:x (:x position)
|
||||
:y (:y position)
|
||||
:width width
|
||||
:height height
|
||||
:metadata {:id id
|
||||
:width width
|
||||
:height height
|
||||
:mtype mtype}
|
||||
:name name
|
||||
:frame-id (:id frame-shape)
|
||||
:parent-id (:id frame-shape)})]
|
||||
[frame-shape [img-shape]]))
|
||||
|
||||
(defn- parse-datauri
|
||||
[data]
|
||||
(let [[mtype b64-data] (str/split data ";base64," 2)
|
||||
mtype (subs mtype (inc (str/index-of mtype ":")))
|
||||
data (-> b64-data bc/str->bytes bc/b64->bytes)]
|
||||
[mtype data]))
|
||||
|
||||
(defn- extract-name
|
||||
[href]
|
||||
(let [query-idx (d/nilv (str/last-index-of href "?") 0)
|
||||
href (if (> query-idx 0) (subs href 0 query-idx) href)
|
||||
filename (->> (str/split href "/") (last))
|
||||
ext-idx (str/last-index-of filename ".")]
|
||||
(if (> ext-idx 0) (subs filename 0 ext-idx) filename)))
|
||||
|
||||
(defn- collect-and-persist-images
|
||||
[svg-data file-id]
|
||||
(letfn [(process-image [{:keys [href] :as item}]
|
||||
(try
|
||||
(let [item (if (str/starts-with? href "data:")
|
||||
(let [[mtype data] (parse-datauri href)
|
||||
size (alength data)
|
||||
path (tmp/tempfile :prefix "penpot.media.download.")
|
||||
written (io/write-to-file! data path :size size)]
|
||||
|
||||
(when (not= written size)
|
||||
(ex/raise :type :internal
|
||||
:code :mismatch-write-size
|
||||
:hint "unexpected state: unable to write to file"))
|
||||
|
||||
(-> item
|
||||
(assoc :size size)
|
||||
(assoc :path path)
|
||||
(assoc :filename "tempfile")
|
||||
(assoc :mtype mtype)))
|
||||
|
||||
(let [result (cmd.media/download-image *system* href)]
|
||||
(-> (merge item result)
|
||||
(assoc :name (extract-name href)))))]
|
||||
|
||||
;; The media processing adds the data to the
|
||||
;; input map and returns it.
|
||||
(media/run {:cmd :info :input item}))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unexpected exception on processing internal image shape (skiping)"
|
||||
:cause cause)
|
||||
(when-not *skip-on-error*
|
||||
(throw cause)))))
|
||||
|
||||
(persist-image [acc {:keys [path size width height mtype href] :as item}]
|
||||
(let [storage (::sto/storage *system*)
|
||||
conn (::db/conn *system*)
|
||||
hash (sto/calculate-hash path)
|
||||
content (-> (sto/content path size)
|
||||
(sto/wrap-with-hash hash))
|
||||
params {::sto/content content
|
||||
::sto/deduplicate? true
|
||||
::sto/touched-at (:ts item)
|
||||
:content-type mtype
|
||||
:bucket "file-media-object"}
|
||||
image (sto/put-object! storage params)
|
||||
fmo-id (uuid/next)]
|
||||
|
||||
(db/exec-one! conn
|
||||
[cmd.media/sql:create-file-media-object
|
||||
fmo-id
|
||||
file-id true (:name item "image")
|
||||
(:id image)
|
||||
nil
|
||||
width
|
||||
height
|
||||
mtype])
|
||||
|
||||
(assoc acc href {:id fmo-id
|
||||
:mtype mtype
|
||||
:width width
|
||||
:height height})))]
|
||||
|
||||
(let [images (->> (csvg/collect-images svg-data)
|
||||
(transduce (keep process-image)
|
||||
(completing persist-image) {}))]
|
||||
(assoc svg-data :image-data images))))
|
||||
|
||||
(defn- get-svg-content
|
||||
[id]
|
||||
(let [storage (::sto/storage *system*)
|
||||
conn (::db/conn *system*)
|
||||
fmobject (db/get conn :file-media-object {:id id})
|
||||
sobject (sto/get-object storage (:media-id fmobject))]
|
||||
|
||||
(with-open [stream (sto/get-object-data storage sobject)]
|
||||
(slurp stream))))
|
||||
|
||||
(defn- create-shapes-for-svg
|
||||
[{:keys [id] :as mobj} file-id objects frame-id position]
|
||||
(let [svg-text (get-svg-content id)
|
||||
|
||||
optimizer (::csvg/optimizer *system*)
|
||||
svg-text (csvg/optimize optimizer svg-text)
|
||||
|
||||
svg-data (-> (csvg/parse svg-text)
|
||||
(assoc :name (:name mobj))
|
||||
(collect-and-persist-images file-id))]
|
||||
|
||||
(sbuilder/create-svg-shapes svg-data position objects frame-id frame-id #{} false)))
|
||||
|
||||
(defn- process-media-object
|
||||
[fdata page-id frame-id mobj position]
|
||||
(let [page (ctpl/get-page fdata page-id)
|
||||
file-id (get fdata :id)
|
||||
|
||||
[shape children]
|
||||
(if (= (:mtype mobj) "image/svg+xml")
|
||||
(create-shapes-for-svg mobj file-id (:objects page) frame-id position)
|
||||
(create-shapes-for-bitmap mobj frame-id position))
|
||||
|
||||
shape (assoc shape :name (-> "Graphics"
|
||||
(cfh/merge-path-item (:path mobj))
|
||||
(cfh/merge-path-item (:name mobj))))
|
||||
|
||||
changes
|
||||
(-> (fcb/empty-changes nil)
|
||||
(fcb/set-save-undo? false)
|
||||
(fcb/with-page page)
|
||||
(fcb/with-objects (:objects page))
|
||||
(fcb/with-library-data fdata)
|
||||
(fcb/delete-media (:id mobj))
|
||||
(fcb/add-objects (cons shape children)))
|
||||
|
||||
;; NOTE: this is a workaround for `generate-add-component`, it
|
||||
;; is needed because that function always starts from empty
|
||||
;; changes; so in this case we need manually add all shapes to
|
||||
;; the page and then use that page for the
|
||||
;; `generate-add-component` function
|
||||
page
|
||||
(reduce (fn [page shape]
|
||||
(ctst/add-shape (:id shape)
|
||||
shape
|
||||
page
|
||||
frame-id
|
||||
frame-id
|
||||
nil
|
||||
true))
|
||||
page
|
||||
(cons shape children))
|
||||
|
||||
[_ _ changes2]
|
||||
(cflh/generate-add-component nil
|
||||
[shape]
|
||||
(:objects page)
|
||||
(:id page)
|
||||
file-id
|
||||
true
|
||||
nil
|
||||
cfsh/prepare-create-artboard-from-selection)
|
||||
changes (fcb/concat-changes changes changes2)]
|
||||
|
||||
(:redo-changes changes)))
|
||||
|
||||
(defn- create-media-grid
|
||||
[fdata page-id frame-id grid media-group]
|
||||
(let [factory (px/thread-factory :virtual true)
|
||||
executor (px/fixed-executor :parallelism 10 :factory factory)
|
||||
process (fn [mobj position]
|
||||
(let [position (gpt/add position (gpt/point grid-gap grid-gap))
|
||||
tp1 (dt/tpoint)]
|
||||
(try
|
||||
(process-media-object fdata page-id frame-id mobj position)
|
||||
(catch Throwable cause
|
||||
(l/wrn :hint "unable to process file media object (skiping)"
|
||||
:file-id (str (:id fdata))
|
||||
:id (str (:id mobj))
|
||||
:cause cause)
|
||||
|
||||
(if-not *skip-on-error*
|
||||
(throw cause)
|
||||
nil))
|
||||
(finally
|
||||
(l/trc :hint "graphic processed"
|
||||
:file-id (str (:id fdata))
|
||||
:media-id (str (:id mobj))
|
||||
:elapsed (dt/format-duration (tp1)))))))]
|
||||
(try
|
||||
(->> (d/zip media-group grid)
|
||||
(map (fn [[mobj position]]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :graphics
|
||||
:name (:name mobj)})
|
||||
(px/submit! executor (partial process mobj position))))
|
||||
(reduce (fn [fdata promise]
|
||||
(if-let [changes (deref promise)]
|
||||
(-> (assoc-in fdata [:options :components-v2] true)
|
||||
(cp/process-changes changes false))
|
||||
fdata))
|
||||
fdata))
|
||||
(finally
|
||||
(pu/close! executor)))))
|
||||
|
||||
(defn- migrate-graphics
|
||||
[fdata]
|
||||
(sse/tap {:type :migration-progress
|
||||
:section :graphics})
|
||||
(if (empty? (:media fdata))
|
||||
fdata
|
||||
(let [[fdata page-id start-pos]
|
||||
(ctf/get-or-add-library-page fdata frame-gap)
|
||||
|
||||
media (->> (vals (:media fdata))
|
||||
(map (fn [{:keys [width height] :as media}]
|
||||
(let [points (-> (grc/make-rect 0 0 width height)
|
||||
(grc/rect->points))]
|
||||
(assoc media :points points)))))
|
||||
|
||||
groups (get-asset-groups media "Graphics")]
|
||||
|
||||
(let [total (count media)]
|
||||
(some-> *stats* (swap! update :processed/graphics (fnil + 0) total))
|
||||
(some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total))
|
||||
(some-> *file-stats* (swap! assoc :processed/graphics total)))
|
||||
|
||||
(loop [groups (seq groups)
|
||||
fdata fdata
|
||||
position start-pos]
|
||||
(if (empty? groups)
|
||||
fdata
|
||||
(let [[group-name assets] (first groups)
|
||||
grid (ctst/generate-shape-grid assets position grid-gap)
|
||||
{:keys [width height]} (meta grid)
|
||||
frame (create-frame group-name position width height)
|
||||
fdata (ctpl/update-page fdata
|
||||
page-id
|
||||
#(ctst/add-shape (:id frame)
|
||||
frame
|
||||
%
|
||||
(:id frame)
|
||||
(:id frame)
|
||||
nil
|
||||
true))]
|
||||
(recur (next groups)
|
||||
(create-media-grid fdata page-id (:id frame) grid assets)
|
||||
(gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap))))))))))
|
||||
|
||||
(defn- migrate-fdata
|
||||
[fdata libs]
|
||||
(let [migrated? (dm/get-in fdata [:options :components-v2])]
|
||||
(if migrated?
|
||||
fdata
|
||||
(let [fdata (migrate-components fdata libs)
|
||||
fdata (migrate-graphics fdata)]
|
||||
(update fdata :options assoc :components-v2 true)))))
|
||||
|
||||
(defn- get-file
|
||||
[system id]
|
||||
(binding [pmap/*load-fn* (partial fdata/load-pointer system id)]
|
||||
(-> (files/get-file system id :migrate? false)
|
||||
(update :data assoc :id id)
|
||||
(update :data fdata/process-pointers deref)
|
||||
(fmg/migrate-file))))
|
||||
|
||||
(defn- validate-file!
|
||||
[file libs throw-on-validate?]
|
||||
(try
|
||||
(cfv/validate-file! file libs)
|
||||
(cfv/validate-file-schema! file)
|
||||
(catch Throwable cause
|
||||
(if throw-on-validate?
|
||||
(throw cause)
|
||||
(l/wrn :hint "migrate:file:validation-error" :cause cause)))))
|
||||
|
||||
(defn- process-file
|
||||
[{:keys [::db/conn] :as system} id & {:keys [validate? throw-on-validate?]}]
|
||||
(let [file (get-file system id)
|
||||
|
||||
libs (->> (files/get-file-libraries conn id)
|
||||
(into [file] (comp (map :id) (map (partial get-file system))))
|
||||
(d/index-by :id))
|
||||
|
||||
file (-> file
|
||||
(update :data migrate-fdata libs)
|
||||
(update :features conj "components/v2"))
|
||||
|
||||
_ (when validate?
|
||||
(validate-file! file libs throw-on-validate?))
|
||||
|
||||
file (if (contains? (:features file) "fdata/objects-map")
|
||||
(fdata/enable-objects-map file)
|
||||
file)
|
||||
|
||||
file (if (contains? (:features file) "fdata/pointer-map")
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)]
|
||||
(let [file (fdata/enable-pointer-map file)]
|
||||
(fdata/persist-pointers! system id)
|
||||
file))
|
||||
file)]
|
||||
|
||||
(db/update! conn :file
|
||||
{:data (blob/encode (:data file))
|
||||
:features (db/create-array conn "text" (:features file))
|
||||
:revn (:revn file)}
|
||||
{:id (:id file)}
|
||||
{::db/return-keys? false})
|
||||
|
||||
(dissoc file :data)))
|
||||
|
||||
(defn migrate-file!
|
||||
[system file-id & {:keys [validate? throw-on-validate?]}]
|
||||
(let [tpoint (dt/tpoint)
|
||||
file-id (if (string? file-id)
|
||||
(parse-uuid file-id)
|
||||
file-id)]
|
||||
(binding [*file-stats* (atom {})]
|
||||
(try
|
||||
(l/dbg :hint "migrate:file:start" :file-id (str file-id))
|
||||
|
||||
(let [system (update system ::sto/storage media/configure-assets-storage)]
|
||||
(db/tx-run! system
|
||||
(fn [system]
|
||||
(binding [*system* system]
|
||||
(fsnap/take-file-snapshot! system {:file-id file-id :label "migration/components-v2"})
|
||||
(process-file system file-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? throw-on-validate?)))))
|
||||
|
||||
(finally
|
||||
(let [elapsed (tpoint)
|
||||
components (get @*file-stats* :processed/components 0)
|
||||
graphics (get @*file-stats* :processed/graphics 0)]
|
||||
|
||||
(l/dbg :hint "migrate:file:end"
|
||||
:file-id (str file-id)
|
||||
:graphics graphics
|
||||
:components components
|
||||
:elapsed (dt/format-duration elapsed))
|
||||
|
||||
(some-> *stats* (swap! update :processed/files (fnil inc 0)))
|
||||
(some-> *team-stats* (swap! update :processed/files (fnil inc 0)))))))))
|
||||
|
||||
(defn migrate-team!
|
||||
[system team-id & {:keys [validate? throw-on-validate?]}]
|
||||
(let [tpoint (dt/tpoint)
|
||||
team-id (if (string? team-id)
|
||||
(parse-uuid team-id)
|
||||
team-id)]
|
||||
(l/dbg :hint "migrate:team:start" :team-id (dm/str team-id))
|
||||
(binding [*team-stats* (atom {})]
|
||||
(try
|
||||
;; We execute this out of transaction because we want this
|
||||
;; change to be visible to all other sessions before starting
|
||||
;; the migration
|
||||
(let [sql (str "UPDATE team SET features = "
|
||||
" array_append(features, 'ephimeral/v2-migration') "
|
||||
" WHERE id = ?")]
|
||||
(db/exec-one! system [sql team-id]))
|
||||
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
;; Lock the team
|
||||
(db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
|
||||
|
||||
(let [{:keys [features] :as team} (-> (db/get conn :team {:id team-id})
|
||||
(update :features db/decode-pgarray #{}))]
|
||||
|
||||
(if (contains? features "components/v2")
|
||||
(l/dbg :hint "team already migrated")
|
||||
(let [sql (str/concat
|
||||
"SELECT f.id FROM file AS f "
|
||||
" JOIN project AS p ON (p.id = f.project_id) "
|
||||
"WHERE p.team_id = ? AND f.deleted_at IS NULL AND p.deleted_at IS NULL "
|
||||
"FOR UPDATE")]
|
||||
|
||||
(doseq [file-id (->> (db/exec! conn [sql team-id])
|
||||
(map :id))]
|
||||
(migrate-file! system file-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? throw-on-validate?))
|
||||
|
||||
(let [features (-> features
|
||||
(disj "ephimeral/v2-migration")
|
||||
(conj "components/v2")
|
||||
(conj "layout/grid")
|
||||
(conj "styles/v2"))]
|
||||
(db/update! conn :team
|
||||
{:features (db/create-array conn "text" features)}
|
||||
{:id team-id})))))))
|
||||
(finally
|
||||
(some-> *semaphore* ps/release!)
|
||||
(let [elapsed (tpoint)]
|
||||
(some-> *stats* (swap! update :processed/teams (fnil inc 0)))
|
||||
|
||||
;; We execute this out of transaction because we want this
|
||||
;; change to be visible to all other sessions before starting
|
||||
;; the migration
|
||||
(let [sql (str "UPDATE team SET features = "
|
||||
" array_remove(features, 'ephimeral/v2-migration') "
|
||||
" WHERE id = ?")]
|
||||
(db/exec-one! system [sql team-id]))
|
||||
|
||||
(let [components (get @*team-stats* :processed/components 0)
|
||||
graphics (get @*team-stats* :processed/graphics 0)
|
||||
files (get @*team-stats* :processed/files 0)]
|
||||
(l/dbg :hint "migrate:team:end"
|
||||
:team-id (dm/str team-id)
|
||||
:files files
|
||||
:components components
|
||||
:graphics graphics
|
||||
:elapsed (dt/format-duration elapsed)))))))))
|
||||
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.features.fdata
|
||||
"A `fdata/*` related feature migration helpers"
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.db :as db]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.objects-map :as omap]
|
||||
[app.util.pointer-map :as pmap]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; OBJECTS-MAP
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn enable-objects-map
|
||||
[file]
|
||||
(let [update-fn #(d/update-when % :objects omap/wrap)]
|
||||
(-> file
|
||||
(update :data (fn [fdata]
|
||||
(-> fdata
|
||||
(update :pages-index update-vals update-fn)
|
||||
(update :components update-vals update-fn))))
|
||||
(update :features conj "fdata/objects-map"))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; POINTER-MAP
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn load-pointer
|
||||
"A database loader pointer helper"
|
||||
[system file-id id]
|
||||
(let [{:keys [content]} (db/get system :file-data-fragment
|
||||
{:id id :file-id file-id}
|
||||
{::db/columns [:content]
|
||||
::db/check-deleted? false})]
|
||||
(when-not content
|
||||
(ex/raise :type :internal
|
||||
:code :fragment-not-found
|
||||
:hint "fragment not found"
|
||||
:file-id file-id
|
||||
:fragment-id id))
|
||||
|
||||
(blob/decode content)))
|
||||
|
||||
(defn persist-pointers!
|
||||
"Given a database connection and the final file-id, persist all
|
||||
pointers to the underlying storage (the database)."
|
||||
[system file-id]
|
||||
(doseq [[id item] @pmap/*tracked*]
|
||||
(when (pmap/modified? item)
|
||||
(l/trc :hint "persist pointer" :file-id (str file-id) :id (str id))
|
||||
(let [content (-> item deref blob/encode)]
|
||||
(db/insert! system :file-data-fragment
|
||||
{:id id
|
||||
:file-id file-id
|
||||
:content content})))))
|
||||
|
||||
(defn process-pointers
|
||||
"Apply a function to all pointers on the file. Usuly used for
|
||||
dereference the pointer to a plain value before some processing."
|
||||
[fdata update-fn]
|
||||
(cond-> fdata
|
||||
(contains? fdata :pages-index)
|
||||
(update :pages-index process-pointers update-fn)
|
||||
|
||||
:always
|
||||
(update-vals (fn [val]
|
||||
(if (pmap/pointer-map? val)
|
||||
(update-fn val)
|
||||
val)))))
|
||||
|
||||
(defn get-used-pointer-ids
|
||||
"Given a file, return all pointer ids used in the data."
|
||||
[fdata]
|
||||
(->> (concat (vals fdata)
|
||||
(vals (:pages-index fdata)))
|
||||
(into #{} (comp (filter pmap/pointer-map?)
|
||||
(map pmap/get-id)))))
|
||||
|
||||
(defn enable-pointer-map
|
||||
"Enable the fdata/pointer-map feature on the file."
|
||||
[file]
|
||||
(-> file
|
||||
(update :data (fn [fdata]
|
||||
(-> fdata
|
||||
(update :pages-index update-vals pmap/wrap)
|
||||
(update :components pmap/wrap))))
|
||||
|
||||
(update :features conj "fdata/pointer-map")))
|
||||
@@ -23,14 +23,15 @@
|
||||
[app.metrics :as mtx]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.doc :as-alias rpc.doc]
|
||||
[app.worker :as wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec :as px]
|
||||
[reitit.core :as r]
|
||||
[reitit.middleware :as rr]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as-alias rres]
|
||||
[yetti.adapter :as yt]))
|
||||
[yetti.adapter :as yt]
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as-alias yrs]))
|
||||
|
||||
(declare router-handler)
|
||||
|
||||
@@ -62,7 +63,8 @@
|
||||
::max-multipart-body-size
|
||||
::router
|
||||
::handler
|
||||
::io-threads]))
|
||||
::io-threads
|
||||
::wrk/executor]))
|
||||
|
||||
(defmethod ig/init-key ::server
|
||||
[_ {:keys [::handler ::router ::host ::port] :as cfg}]
|
||||
@@ -73,9 +75,11 @@
|
||||
:http/max-multipart-body-size (::max-multipart-body-size cfg)
|
||||
:xnio/io-threads (or (::io-threads cfg)
|
||||
(max 3 (px/get-available-processors)))
|
||||
:xnio/dispatch :virtual
|
||||
:ring/compat :ring2
|
||||
:socket/backlog 4069}
|
||||
:xnio/worker-threads (or (::worker-threads cfg)
|
||||
(max 6 (px/get-available-processors)))
|
||||
:xnio/dispatch true
|
||||
:socket/backlog 4069
|
||||
:ring/async true}
|
||||
|
||||
handler (cond
|
||||
(some? router)
|
||||
@@ -98,13 +102,13 @@
|
||||
(yt/stop! server))
|
||||
|
||||
(defn- not-found-handler
|
||||
[_]
|
||||
{::rres/status 404})
|
||||
[_ respond _]
|
||||
(respond {::yrs/status 404}))
|
||||
|
||||
(defn- router-handler
|
||||
[router]
|
||||
(letfn [(resolve-handler [request]
|
||||
(if-let [match (r/match-by-path router (rreq/path request))]
|
||||
(if-let [match (r/match-by-path router (yrq/path request))]
|
||||
(let [params (:path-params match)
|
||||
result (:result match)
|
||||
handler (or (:handler result) not-found-handler)
|
||||
@@ -116,15 +120,18 @@
|
||||
(let [{:keys [body] :as response} (errors/handle cause request)]
|
||||
(cond-> response
|
||||
(map? body)
|
||||
(-> (update ::rres/headers assoc "content-type" "application/transit+json")
|
||||
(assoc ::rres/body (t/encode-str body {:type :json-verbose}))))))]
|
||||
(-> (update ::yrs/headers assoc "content-type" "application/transit+json")
|
||||
(assoc ::yrs/body (t/encode-str body {:type :json-verbose}))))))]
|
||||
|
||||
(fn [request]
|
||||
(let [handler (resolve-handler request)]
|
||||
(try
|
||||
(handler)
|
||||
(catch Throwable cause
|
||||
(on-error cause request)))))))
|
||||
(fn [request respond _]
|
||||
(let [handler (resolve-handler request)
|
||||
exchange (yrq/exchange request)]
|
||||
(handler
|
||||
(fn [response]
|
||||
(yt/dispatch! exchange (partial respond response)))
|
||||
(fn [cause]
|
||||
(let [response (on-error cause request)]
|
||||
(yt/dispatch! exchange (partial respond response)))))))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HTTP ROUTER
|
||||
@@ -153,7 +160,8 @@
|
||||
[session/soft-auth cfg]
|
||||
[actoken/soft-auth cfg]
|
||||
[mw/errors errors/handle]
|
||||
[mw/restrict-methods]]}
|
||||
[mw/restrict-methods]
|
||||
[mw/with-dispatch :vthread]]}
|
||||
|
||||
(::mtx/routes cfg)
|
||||
(::assets/routes cfg)
|
||||
|
||||
@@ -11,13 +11,13 @@
|
||||
[app.db :as db]
|
||||
[app.main :as-alias main]
|
||||
[app.tokens :as tokens]
|
||||
[ring.request :as rreq]))
|
||||
[yetti.request :as yrq]))
|
||||
|
||||
(def header-re #"^Token\s+(.*)")
|
||||
|
||||
(defn- get-token
|
||||
[request]
|
||||
(some->> (rreq/get-header request "authorization")
|
||||
(some->> (yrq/get-header request "authorization")
|
||||
(re-matches header-re)
|
||||
(second)))
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"SELECT perms, profile_id, expires_at
|
||||
FROM access_token
|
||||
WHERE id = ?
|
||||
AND (expires_at IS NULL
|
||||
AND (expires_at IS NULL
|
||||
OR (expires_at > now()));")
|
||||
|
||||
(defn- get-token-data
|
||||
@@ -54,8 +54,9 @@
|
||||
(l/trace :hint "exception on decoding malformed token" :cause cause)
|
||||
request)))]
|
||||
|
||||
(fn [request]
|
||||
(handler (handle-request request)))))
|
||||
(fn [request respond raise]
|
||||
(let [request (handle-request request)]
|
||||
(handler request respond raise)))))
|
||||
|
||||
(defn- wrap-authz
|
||||
"Authorization middleware, will be executed synchronously on vthread."
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.response :as-alias yrs]))
|
||||
|
||||
(def ^:private cache-max-age
|
||||
(dt/duration {:hours 24}))
|
||||
@@ -37,11 +37,11 @@
|
||||
(defn- serve-object-from-s3
|
||||
[{:keys [::sto/storage] :as cfg} obj]
|
||||
(let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
|
||||
{::rres/status 307
|
||||
::rres/headers {"location" (str url)
|
||||
"x-host" (cond-> host port (str ":" port))
|
||||
"x-mtype" (-> obj meta :content-type)
|
||||
"cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
|
||||
{::yrs/status 307
|
||||
::yrs/headers {"location" (str url)
|
||||
"x-host" (cond-> host port (str ":" port))
|
||||
"x-mtype" (-> obj meta :content-type)
|
||||
"cache-control" (str "max-age=" (inst-ms cache-max-age))}}))
|
||||
|
||||
(defn- serve-object-from-fs
|
||||
[{:keys [::path]} obj]
|
||||
@@ -51,8 +51,8 @@
|
||||
headers {"x-accel-redirect" (:path purl)
|
||||
"content-type" (:content-type mdata)
|
||||
"cache-control" (str "max-age=" (inst-ms cache-max-age))}]
|
||||
{::rres/status 204
|
||||
::rres/headers headers}))
|
||||
{::yrs/status 204
|
||||
::yrs/headers headers}))
|
||||
|
||||
(defn- serve-object
|
||||
"Helper function that returns the appropriate response depending on
|
||||
@@ -70,7 +70,7 @@
|
||||
obj (sto/get-object storage id)]
|
||||
(if obj
|
||||
(serve-object cfg obj)
|
||||
{::rres/status 404})))
|
||||
{::yrs/status 404})))
|
||||
|
||||
(defn- generic-handler
|
||||
"A generic handler helper/common code for file-media based handlers."
|
||||
@@ -81,7 +81,7 @@
|
||||
sobj (sto/get-object storage (kf mobj))]
|
||||
(if sobj
|
||||
(serve-object cfg sobj)
|
||||
{::rres/status 404})))
|
||||
{::yrs/status 404})))
|
||||
|
||||
(defn file-objects-handler
|
||||
"Handler that serves storage objects by file media id."
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
[integrant.core :as ig]
|
||||
[jsonista.core :as j]
|
||||
[promesa.exec :as px]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as-alias yrs]))
|
||||
|
||||
(declare parse-json)
|
||||
(declare handle-request)
|
||||
@@ -31,14 +31,15 @@
|
||||
(defmethod ig/pre-init-spec ::routes [_]
|
||||
(s/keys :req [::http/client
|
||||
::main/props
|
||||
::db/pool]))
|
||||
::db/pool
|
||||
::wrk/executor]))
|
||||
|
||||
(defmethod ig/init-key ::routes
|
||||
[_ cfg]
|
||||
(letfn [(handler [request]
|
||||
(let [data (-> request rreq/body slurp)]
|
||||
(px/run! :vthread (partial handle-request cfg data)))
|
||||
{::rres/status 200})]
|
||||
[_ {:keys [::wrk/executor] :as cfg}]
|
||||
(letfn [(handler [request respond _]
|
||||
(let [data (-> request yrq/body slurp)]
|
||||
(px/run! executor #(handle-request cfg data)))
|
||||
(respond {::yrs/status 200}))]
|
||||
["/sns" {:handler handler
|
||||
:allowed-methods #{:post}}]))
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"Http client abstraction layer."
|
||||
(:require
|
||||
[app.common.spec :as us]
|
||||
[app.worker :as wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[java-http-clj.core :as http]
|
||||
@@ -20,11 +21,12 @@
|
||||
(s/keys :req [::client]))
|
||||
|
||||
(defmethod ig/pre-init-spec ::client [_]
|
||||
(s/keys :req []))
|
||||
(s/keys :req [::wrk/executor]))
|
||||
|
||||
(defmethod ig/init-key ::client
|
||||
[_ _]
|
||||
(http/build-client {:connect-timeout 30000 ;; 10s
|
||||
[_ {:keys [::wrk/executor] :as cfg}]
|
||||
(http/build-client {:executor executor
|
||||
:connect-timeout 30000 ;; 10s
|
||||
:follow-redirects :always}))
|
||||
|
||||
(defn send!
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
(ns app.http.debug
|
||||
(:refer-clojure :exclude [error-handler])
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
@@ -15,12 +14,9 @@
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http.session :as session]
|
||||
[app.main :as-alias main]
|
||||
[app.rpc.commands.auth :as auth]
|
||||
[app.rpc.commands.binfile :as binf]
|
||||
[app.rpc.commands.files-create :refer [create-file]]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.srepl.helpers :as srepl]
|
||||
[app.storage :as-alias sto]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.template :as tmpl]
|
||||
@@ -32,41 +28,55 @@
|
||||
[integrant.core :as ig]
|
||||
[markdown.core :as md]
|
||||
[markdown.transformers :as mdt]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as rres]))
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
;; (selmer.parser/cache-off!)
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HELPERS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn authorized?
|
||||
[pool {:keys [::session/profile-id]}]
|
||||
(or (= "devenv" (cf/get :host))
|
||||
(let [profile (ex/ignoring (profile/get-profile pool profile-id))
|
||||
admins (or (cf/get :admins) #{})]
|
||||
(contains? admins (:email profile)))))
|
||||
|
||||
(defn prepare-response
|
||||
[body]
|
||||
(let [headers {"content-type" "application/transit+json"}]
|
||||
{::yrs/status 200
|
||||
::yrs/body body
|
||||
::yrs/headers headers}))
|
||||
|
||||
(defn prepare-download-response
|
||||
[body filename]
|
||||
(let [headers {"content-disposition" (str "attachment; filename=" filename)
|
||||
"content-type" "application/octet-stream"}]
|
||||
{::yrs/status 200
|
||||
::yrs/body body
|
||||
::yrs/headers headers}))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; INDEX
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn index-handler
|
||||
[_cfg _request]
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/html"}
|
||||
::rres/body (-> (io/resource "app/templates/debug.tmpl")
|
||||
(tmpl/render {}))})
|
||||
[{:keys [::db/pool]} request]
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "text/html"}
|
||||
::yrs/body (-> (io/resource "app/templates/debug.tmpl")
|
||||
(tmpl/render {}))})
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; FILE CHANGES
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn prepare-response
|
||||
[body]
|
||||
(let [headers {"content-type" "application/transit+json"}]
|
||||
{::rres/status 200
|
||||
::rres/body body
|
||||
::rres/headers headers}))
|
||||
|
||||
(defn prepare-download-response
|
||||
[body filename]
|
||||
(let [headers {"content-disposition" (str "attachment; filename=" filename)
|
||||
"content-type" "application/octet-stream"}]
|
||||
{::rres/status 200
|
||||
::rres/body body
|
||||
::rres/headers headers}))
|
||||
|
||||
(def sql:retrieve-range-of-changes
|
||||
"select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
|
||||
|
||||
@@ -75,6 +85,10 @@
|
||||
|
||||
(defn- retrieve-file-data
|
||||
[{:keys [::db/pool]} {:keys [params ::session/profile-id] :as request}]
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
|
||||
(let [file-id (some-> params :file-id parse-uuid)
|
||||
revn (some-> params :revn parse-long)
|
||||
filename (str file-id)]
|
||||
@@ -97,18 +111,15 @@
|
||||
|
||||
(contains? params :clone)
|
||||
(let [profile (profile/get-profile pool profile-id)
|
||||
project-id (:default-project-id profile)]
|
||||
|
||||
(db/run! pool (fn [{:keys [::db/conn]}]
|
||||
(create-file conn {:id file-id
|
||||
:name (str "Cloned file: " filename)
|
||||
:project-id project-id
|
||||
:profile-id profile-id})
|
||||
(db/update! conn :file
|
||||
{:data data}
|
||||
{:id file-id})
|
||||
{::rres/status 201
|
||||
::rres/body "OK CREATED"})))
|
||||
project-id (:default-project-id profile)
|
||||
data (blob/decode data)]
|
||||
(create-file pool {:id (uuid/next)
|
||||
:name (str "Cloned file: " filename)
|
||||
:project-id project-id
|
||||
:profile-id profile-id
|
||||
:data data})
|
||||
{::yrs/status 201
|
||||
::yrs/body "OK CREATED"})
|
||||
|
||||
:else
|
||||
(prepare-response (blob/decode data))))))
|
||||
@@ -122,41 +133,38 @@
|
||||
[{:keys [::db/pool]} {:keys [::session/profile-id params] :as request}]
|
||||
(let [profile (profile/get-profile pool profile-id)
|
||||
project-id (:default-project-id profile)
|
||||
data (some-> params :file :path io/read-as-bytes)]
|
||||
data (some-> params :file :path io/read-as-bytes blob/decode)]
|
||||
|
||||
(if (and data project-id)
|
||||
(let [fname (str "Imported file *: " (dt/now))
|
||||
reuse-id? (contains? params :reuseid)
|
||||
file-id (or (and reuse-id? (ex/ignoring (-> params :file :filename parse-uuid)))
|
||||
(uuid/next))]
|
||||
(let [fname (str "Imported file *: " (dt/now))
|
||||
overwrite? (contains? params :overwrite?)
|
||||
file-id (or (and overwrite? (ex/ignoring (-> params :file :filename parse-uuid)))
|
||||
(uuid/next))]
|
||||
|
||||
(if (and reuse-id? file-id
|
||||
(if (and overwrite? file-id
|
||||
(is-file-exists? pool file-id))
|
||||
(do
|
||||
(db/update! pool :file
|
||||
{:data data
|
||||
:deleted-at nil}
|
||||
{:data (blob/encode data)}
|
||||
{:id file-id})
|
||||
{::rres/status 200
|
||||
::rres/body "OK UPDATED"})
|
||||
{::yrs/status 200
|
||||
::yrs/body "OK UPDATED"})
|
||||
|
||||
(db/run! pool (fn [{:keys [::db/conn]}]
|
||||
(create-file conn {:id file-id
|
||||
:name fname
|
||||
:project-id project-id
|
||||
:profile-id profile-id})
|
||||
(db/update! conn :file
|
||||
{:data data}
|
||||
{:id file-id})
|
||||
{::rres/status 201
|
||||
::rres/body "OK CREATED"}))))
|
||||
(do
|
||||
(create-file pool {:id file-id
|
||||
:name fname
|
||||
:project-id project-id
|
||||
:profile-id profile-id
|
||||
:data data})
|
||||
{::yrs/status 201
|
||||
::yrs/body "OK CREATED"})))
|
||||
|
||||
{::rres/status 500
|
||||
::rres/body "ERROR"})))
|
||||
{::yrs/status 500
|
||||
::yrs/body "ERROR"})))
|
||||
|
||||
(defn file-data-handler
|
||||
[cfg request]
|
||||
(case (rreq/method request)
|
||||
(case (yrq/method request)
|
||||
:get (retrieve-file-data cfg request)
|
||||
:post (upload-file-data cfg request)
|
||||
(ex/raise :type :http
|
||||
@@ -164,6 +172,10 @@
|
||||
|
||||
(defn file-changes-handler
|
||||
[{:keys [::db/pool]} {:keys [params] :as request}]
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
|
||||
(letfn [(retrieve-changes [file-id revn]
|
||||
(if (str/includes? revn ":")
|
||||
(let [[start end] (->> (str/split revn #":")
|
||||
@@ -226,40 +238,46 @@
|
||||
(-> (io/resource "app/templates/error-report.v2.tmpl")
|
||||
(tmpl/render report)))
|
||||
|
||||
(render-template-v3 [{:keys [content id created-at]}]
|
||||
(render-template-v3 [{report :content}]
|
||||
(-> (io/resource "app/templates/error-report.v3.tmpl")
|
||||
(tmpl/render (-> content
|
||||
(assoc :id id)
|
||||
(assoc :created-at (dt/format-instant created-at :rfc1123))))))]
|
||||
(tmpl/render report)))
|
||||
]
|
||||
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
|
||||
(if-let [report (get-report request)]
|
||||
(let [result (case (:version report)
|
||||
1 (render-template-v1 report)
|
||||
2 (render-template-v2 report)
|
||||
3 (render-template-v3 report))]
|
||||
{::rres/status 200
|
||||
::rres/body result
|
||||
::rres/headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"}})
|
||||
{::rres/status 404
|
||||
::rres/body "not found"})))
|
||||
{::yrs/status 200
|
||||
::yrs/body result
|
||||
::yrs/headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"}})
|
||||
{::yrs/status 404
|
||||
::yrs/body "not found"})))
|
||||
|
||||
(def sql:error-reports
|
||||
"SELECT id, created_at,
|
||||
content->>'~:hint' AS hint
|
||||
FROM server_error_report
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 200")
|
||||
LIMIT 100")
|
||||
|
||||
(defn error-list-handler
|
||||
[{:keys [::db/pool]} _request]
|
||||
[{:keys [::db/pool]} request]
|
||||
(when-not (authorized? pool request)
|
||||
(ex/raise :type :authentication
|
||||
:code :only-admins-allowed))
|
||||
(let [items (->> (db/exec! pool [sql:error-reports])
|
||||
(map #(update % :created-at dt/format-instant :rfc1123)))]
|
||||
{::rres/status 200
|
||||
::rres/body (-> (io/resource "app/templates/error-list.tmpl")
|
||||
(tmpl/render {:items items}))
|
||||
::rres/headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"}}))
|
||||
{::yrs/status 200
|
||||
::yrs/body (-> (io/resource "app/templates/error-list.tmpl")
|
||||
(tmpl/render {:items items}))
|
||||
::yrs/headers {"content-type" "text/html; charset=utf-8"
|
||||
"x-robots-tag" "noindex"}}))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; EXPORT/IMPORT
|
||||
@@ -295,14 +313,14 @@
|
||||
::binf/profile-id profile-id
|
||||
::binf/project-id project-id))
|
||||
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body "OK CLONED"})
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "text/plain"}
|
||||
::yrs/body "OK CLONED"})
|
||||
|
||||
{::rres/status 200
|
||||
::rres/body (io/input-stream path)
|
||||
::rres/headers {"content-type" "application/octet-stream"
|
||||
"content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
|
||||
{::yrs/status 200
|
||||
::yrs/body (io/input-stream path)
|
||||
::yrs/headers {"content-type" "application/octet-stream"
|
||||
"content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))
|
||||
|
||||
|
||||
|
||||
@@ -333,96 +351,9 @@
|
||||
::binf/profile-id profile-id
|
||||
::binf/project-id project-id))
|
||||
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body "OK"}))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; ACTIONS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn- resend-email-notification
|
||||
[{:keys [::db/pool ::main/props] :as cfg} {:keys [params] :as request}]
|
||||
|
||||
(when-not (contains? params :force)
|
||||
(ex/raise :type :validation
|
||||
:code :missing-force
|
||||
:hint "missing force checkbox"))
|
||||
|
||||
(let [profile (some->> params :email (profile/get-profile-by-email pool))]
|
||||
|
||||
(when-not profile
|
||||
(ex/raise :type :validation
|
||||
:code :missing-profile
|
||||
:hint "unable to find profile by email"))
|
||||
|
||||
(cond
|
||||
(contains? params :block)
|
||||
(do
|
||||
(db/update! pool :profile {:is-blocked true} {:id (:id profile)})
|
||||
(db/delete! pool :http-session {:profile-id (:id profile)})
|
||||
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body (str/ffmt "PROFILE '%' BLOCKED" (:email profile))})
|
||||
|
||||
(contains? params :unblock)
|
||||
(do
|
||||
(db/update! pool :profile {:is-blocked false} {:id (:id profile)})
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body (str/ffmt "PROFILE '%' UNBLOCKED" (:email profile))})
|
||||
|
||||
(contains? params :resend)
|
||||
(if (:is-blocked profile)
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body "PROFILE ALREADY BLOCKED"}
|
||||
(do
|
||||
(auth/send-email-verification! pool props profile)
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body (str/ffmt "RESENDED FOR '%'" (:email profile))}))
|
||||
|
||||
:else
|
||||
(do
|
||||
(db/update! pool :profile {:is-active true} {:id (:id profile)})
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body (str/ffmt "PROFILE '%' ACTIVATED" (:email profile))}))))
|
||||
|
||||
|
||||
(defn- reset-file-data-version
|
||||
[cfg {:keys [params] :as request}]
|
||||
(let [file-id (some-> params :file-id d/parse-uuid)
|
||||
version (some-> params :version d/parse-integer)]
|
||||
|
||||
(when-not (contains? params :force)
|
||||
(ex/raise :type :validation
|
||||
:code :missing-force
|
||||
:hint "missing force checkbox"))
|
||||
|
||||
(when (nil? file-id)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-file-id
|
||||
:hint "provided invalid file id"))
|
||||
|
||||
(when (nil? version)
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-version
|
||||
:hint "provided invalid version"))
|
||||
|
||||
(srepl/update-file! cfg
|
||||
:id file-id
|
||||
:update-fn (fn [file]
|
||||
(update file :data assoc :version version))
|
||||
:migrate? false
|
||||
:inc-revn? false
|
||||
:save? true)
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/plain"}
|
||||
::rres/body "OK"}))
|
||||
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "text/plain"}
|
||||
::yrs/body "OK"}))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; OTHER SMALL VIEWS/HANDLERS
|
||||
@@ -433,13 +364,13 @@
|
||||
[{:keys [::db/pool]} _]
|
||||
(try
|
||||
(db/exec-one! pool ["select count(*) as count from server_prop;"])
|
||||
{::rres/status 200
|
||||
::rres/body "OK"}
|
||||
{::yrs/status 200
|
||||
::yrs/body "OK"}
|
||||
(catch Throwable cause
|
||||
(l/warn :hint "unable to execute query on health handler"
|
||||
:cause cause)
|
||||
{::rres/status 503
|
||||
::rres/body "KO"})))
|
||||
{::yrs/status 503
|
||||
::yrs/body "KO"})))
|
||||
|
||||
(defn changelog-handler
|
||||
[_ _]
|
||||
@@ -448,23 +379,16 @@
|
||||
(md->html [text]
|
||||
(md/md-to-html-string text :replacement-transformers (into [transform-emoji] mdt/transformer-vector)))]
|
||||
(if-let [clog (io/resource "changelog.md")]
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/html; charset=utf-8"}
|
||||
::rres/body (-> clog slurp md->html)}
|
||||
{::rres/status 404
|
||||
::rres/body "NOT FOUND"})))
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "text/html; charset=utf-8"}
|
||||
::yrs/body (-> clog slurp md->html)}
|
||||
{::yrs/status 404
|
||||
::yrs/body "NOT FOUND"})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; INIT
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(defn authorized?
|
||||
[pool {:keys [::session/profile-id]}]
|
||||
(or (= "devenv" (cf/get :host))
|
||||
(let [profile (ex/ignoring (profile/get-profile pool profile-id))
|
||||
admins (or (cf/get :admins) #{})]
|
||||
(contains? admins (:email profile)))))
|
||||
|
||||
(def with-authorization
|
||||
{:compile
|
||||
(fn [& _]
|
||||
@@ -488,10 +412,6 @@
|
||||
["/changelog" {:handler (partial changelog-handler cfg)}]
|
||||
["/error/:id" {:handler (partial error-handler cfg)}]
|
||||
["/error" {:handler (partial error-list-handler cfg)}]
|
||||
["/actions/resend-email-verification"
|
||||
{:handler (partial resend-email-notification cfg)}]
|
||||
["/actions/reset-file-data-version"
|
||||
{:handler (partial reset-file-data-version cfg)}]
|
||||
["/file/export" {:handler (partial export-handler cfg)}]
|
||||
["/file/import" {:handler (partial import-handler cfg)}]
|
||||
["/file/data" {:handler (partial file-data-handler cfg)}]
|
||||
|
||||
@@ -9,21 +9,20 @@
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as-alias sm]
|
||||
[app.config :as cf]
|
||||
[app.common.schema :as sm]
|
||||
[app.http :as-alias http]
|
||||
[app.http.access-token :as-alias actoken]
|
||||
[app.http.session :as-alias session]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as rres]))
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
(defn- parse-client-ip
|
||||
[request]
|
||||
(or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
|
||||
(rreq/get-header request "x-real-ip")
|
||||
(rreq/remote-addr request)))
|
||||
(or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
|
||||
(yrq/get-header request "x-real-ip")
|
||||
(yrq/remote-addr request)))
|
||||
|
||||
(defn request->context
|
||||
"Extracts error report relevant context data from request."
|
||||
@@ -31,209 +30,182 @@
|
||||
(let [claims (-> {}
|
||||
(into (::session/token-claims request))
|
||||
(into (::actoken/token-claims request)))]
|
||||
{:request/path (:path request)
|
||||
:request/method (:method request)
|
||||
:request/params (:params request)
|
||||
:request/user-agent (rreq/get-header request "user-agent")
|
||||
:request/ip-addr (parse-client-ip request)
|
||||
:request/profile-id (:uid claims)
|
||||
:version/frontend (or (rreq/get-header request "x-frontend-version") "unknown")
|
||||
:version/backend (:full cf/version)}))
|
||||
|
||||
(defmulti handle-error
|
||||
(fn [cause _ _]
|
||||
(-> cause ex-data :type)))
|
||||
{:path (:path request)
|
||||
:method (:method request)
|
||||
:params (:params request)
|
||||
:ip-addr (parse-client-ip request)
|
||||
:user-agent (yrq/get-header request "user-agent")
|
||||
:profile-id (:uid claims)
|
||||
:version (or (yrq/get-header request "x-frontend-version")
|
||||
"unknown")}))
|
||||
|
||||
(defmulti handle-exception
|
||||
(fn [cause _ _]
|
||||
(class cause)))
|
||||
(fn [err & _rest]
|
||||
(let [edata (ex-data err)]
|
||||
(or (:type edata)
|
||||
(class err)))))
|
||||
|
||||
(defmethod handle-error :authentication
|
||||
[err _ _]
|
||||
{::rres/status 401
|
||||
::rres/body (ex-data err)})
|
||||
(defmethod handle-exception :authentication
|
||||
[err _]
|
||||
{::yrs/status 401
|
||||
::yrs/body (ex-data err)})
|
||||
|
||||
(defmethod handle-error :authorization
|
||||
[err _ _]
|
||||
{::rres/status 403
|
||||
::rres/body (ex-data err)})
|
||||
(defmethod handle-exception :authorization
|
||||
[err _]
|
||||
{::yrs/status 403
|
||||
::yrs/body (ex-data err)})
|
||||
|
||||
(defmethod handle-error :restriction
|
||||
[err _ _]
|
||||
{::rres/status 400
|
||||
::rres/body (ex-data err)})
|
||||
(defmethod handle-exception :restriction
|
||||
[err _]
|
||||
{::yrs/status 400
|
||||
::yrs/body (ex-data err)})
|
||||
|
||||
(defmethod handle-error :rate-limit
|
||||
[err _ _]
|
||||
(defmethod handle-exception :rate-limit
|
||||
[err _]
|
||||
(let [headers (-> err ex-data ::http/headers)]
|
||||
{::rres/status 429
|
||||
::rres/headers headers}))
|
||||
{::yrs/status 429
|
||||
::yrs/headers headers}))
|
||||
|
||||
(defmethod handle-error :concurrency-limit
|
||||
[err _ _]
|
||||
(defmethod handle-exception :concurrency-limit
|
||||
[err _]
|
||||
(let [headers (-> err ex-data ::http/headers)]
|
||||
{::rres/status 429
|
||||
::rres/headers headers}))
|
||||
{::yrs/status 429
|
||||
::yrs/headers headers}))
|
||||
|
||||
(defmethod handle-error :validation
|
||||
[err request parent-cause]
|
||||
(defmethod handle-exception :validation
|
||||
[err _]
|
||||
(let [{:keys [code] :as data} (ex-data err)]
|
||||
(cond
|
||||
(or (= code :spec-validation)
|
||||
(= code :params-validation)
|
||||
(= code :schema-validation)
|
||||
(= code :data-validation))
|
||||
(= code :spec-validation)
|
||||
(let [explain (ex/explain data)]
|
||||
{::rres/status 400
|
||||
::rres/body (-> data
|
||||
(dissoc ::s/problems ::s/value ::s/spec ::sm/explain)
|
||||
(cond-> explain (assoc :explain explain)))})
|
||||
{::yrs/status 400
|
||||
::yrs/body (-> data
|
||||
(dissoc ::s/problems ::s/value)
|
||||
(cond-> explain (assoc :explain explain)))})
|
||||
|
||||
(= code :params-validation)
|
||||
(let [explain (::sm/explain data)
|
||||
payload (sm/humanize-data explain)]
|
||||
{::yrs/status 400
|
||||
::yrs/body (-> data
|
||||
(dissoc ::sm/explain)
|
||||
(assoc :data payload))})
|
||||
|
||||
(= code :request-body-too-large)
|
||||
{::rres/status 413 ::rres/body data}
|
||||
|
||||
(= code :invalid-image)
|
||||
(binding [l/*context* (request->context request)]
|
||||
(let [cause (or parent-cause err)]
|
||||
(l/error :hint "unexpected error on processing image" :cause cause)
|
||||
{::rres/status 400 ::rres/body data}))
|
||||
{::yrs/status 413 ::yrs/body data}
|
||||
|
||||
:else
|
||||
{::rres/status 400 ::rres/body data})))
|
||||
{::yrs/status 400 ::yrs/body data})))
|
||||
|
||||
(defmethod handle-error :assertion
|
||||
[error request parent-cause]
|
||||
(defmethod handle-exception :assertion
|
||||
[error request]
|
||||
(binding [l/*context* (request->context request)]
|
||||
(let [{:keys [code] :as data} (ex-data error)
|
||||
cause (or parent-cause error)]
|
||||
(let [{:keys [code] :as data} (ex-data error)]
|
||||
(cond
|
||||
(= code :data-validation)
|
||||
(let [explain (ex/explain data)]
|
||||
(l/error :hint "data assertion error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> data
|
||||
(dissoc ::sm/explain)
|
||||
(cond-> explain (assoc :explain explain)))}})
|
||||
(let [explain (::sm/explain data)
|
||||
payload (sm/humanize-data explain)]
|
||||
(l/error :hint "Data assertion error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> data
|
||||
(dissoc ::sm/explain)
|
||||
(assoc :data payload))}})
|
||||
|
||||
(= code :spec-validation)
|
||||
(let [explain (ex/explain data)]
|
||||
(l/error :hint "spec assertion error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> data
|
||||
(dissoc ::s/problems ::s/value ::s/spec)
|
||||
(cond-> explain (assoc :explain explain)))}})
|
||||
(l/error :hint "Spec assertion error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :assertion
|
||||
:data (-> data
|
||||
(dissoc ::s/problems ::s/value ::s/spec)
|
||||
(cond-> explain (assoc :explain explain)))}})
|
||||
|
||||
:else
|
||||
(do
|
||||
(l/error :hint "assertion error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :assertion
|
||||
:data data}})))))
|
||||
(l/error :hint "Assertion error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :assertion
|
||||
:data data}})))))
|
||||
|
||||
(defmethod handle-error :not-found
|
||||
[err _ _]
|
||||
{::rres/status 404
|
||||
::rres/body (ex-data err)})
|
||||
|
||||
(defmethod handle-error :internal
|
||||
[error request parent-cause]
|
||||
(defmethod handle-exception :not-found
|
||||
[err _]
|
||||
{::yrs/status 404
|
||||
::yrs/body (ex-data err)})
|
||||
|
||||
(defmethod handle-exception :internal
|
||||
[error request]
|
||||
(binding [l/*context* (request->context request)]
|
||||
(let [cause (or parent-cause error)]
|
||||
(l/error :hint "internal error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
:data (ex-data error)}})))
|
||||
|
||||
(defmethod handle-error :default
|
||||
[error request parent-cause]
|
||||
(let [edata (ex-data error)]
|
||||
;; This is a special case for the idle-in-transaction error;
|
||||
;; when it happens, the connection is automatically closed and
|
||||
;; next-jdbc combines the two errors in a single ex-info. We
|
||||
;; only need the :handling error, because the :rollback error
|
||||
;; will be always "connection closed".
|
||||
(if (and (ex/exception? (:rollback edata))
|
||||
(ex/exception? (:handling edata)))
|
||||
(handle-exception (:handling edata) request error)
|
||||
(handle-exception error request parent-cause))))
|
||||
(l/error :hint "Internal error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
:data (ex-data error)}}))
|
||||
|
||||
(defmethod handle-exception org.postgresql.util.PSQLException
|
||||
[error request parent-cause]
|
||||
(let [state (.getSQLState ^java.sql.SQLException error)
|
||||
cause (or parent-cause error)]
|
||||
[error request]
|
||||
(let [state (.getSQLState ^java.sql.SQLException error)]
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "PSQL error"
|
||||
:cause cause)
|
||||
(l/error :hint "PSQL error" :message (ex-message error) :cause error)
|
||||
(cond
|
||||
(= state "57014")
|
||||
{::rres/status 504
|
||||
::rres/body {:type :server-error
|
||||
:code :statement-timeout
|
||||
:hint (ex-message error)}}
|
||||
{::yrs/status 504
|
||||
::yrs/body {:type :server-error
|
||||
:code :statement-timeout
|
||||
:hint (ex-message error)}}
|
||||
|
||||
(= state "25P03")
|
||||
{::rres/status 504
|
||||
::rres/body {:type :server-error
|
||||
:code :idle-in-transaction-timeout
|
||||
:hint (ex-message error)}}
|
||||
{::yrs/status 504
|
||||
::yrs/body {:type :server-error
|
||||
:code :idle-in-transaction-timeout
|
||||
:hint (ex-message error)}}
|
||||
|
||||
:else
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)
|
||||
:state state}}))))
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)
|
||||
:state state}}))))
|
||||
|
||||
(defmethod handle-exception :default
|
||||
[error request parent-cause]
|
||||
(let [edata (ex-data error)
|
||||
cause (or parent-cause error)]
|
||||
[error request]
|
||||
(let [edata (ex-data error)]
|
||||
(cond
|
||||
;; This means that exception is not a controlled exception.
|
||||
(nil? edata)
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "unexpected error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)}})
|
||||
(l/error :hint "Unexpected error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :unexpected
|
||||
:hint (ex-message error)}})
|
||||
|
||||
;; This is a special case for the idle-in-transaction error;
|
||||
;; when it happens, the connection is automatically closed and
|
||||
;; next-jdbc combines the two errors in a single ex-info. We
|
||||
;; only need the :handling error, because the :rollback error
|
||||
;; will be always "connection closed".
|
||||
(and (ex/exception? (:rollback edata))
|
||||
(ex/exception? (:handling edata)))
|
||||
(handle-exception (:handling edata) request)
|
||||
|
||||
:else
|
||||
(binding [l/*context* (request->context request)]
|
||||
(l/error :hint "unhandled error" :cause cause)
|
||||
{::rres/status 500
|
||||
::rres/body {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
:data edata}}))))
|
||||
|
||||
(defmethod handle-exception java.util.concurrent.CompletionException
|
||||
[cause request _]
|
||||
(let [cause' (ex-cause cause)]
|
||||
(if (ex/error? cause')
|
||||
(handle-error cause' request cause)
|
||||
(handle-exception cause' request cause))))
|
||||
|
||||
(defmethod handle-exception java.util.concurrent.ExecutionException
|
||||
[cause request _]
|
||||
(let [cause' (ex-cause cause)]
|
||||
(if (ex/error? cause')
|
||||
(handle-error cause' request cause)
|
||||
(handle-exception cause' request cause))))
|
||||
(l/error :hint "Unhandled error" :message (ex-message error) :cause error)
|
||||
{::yrs/status 500
|
||||
::yrs/body {:type :server-error
|
||||
:code :unhandled
|
||||
:hint (ex-message error)
|
||||
:data edata}}))))
|
||||
|
||||
(defn handle
|
||||
[cause request]
|
||||
(if (ex/error? cause)
|
||||
(handle-error cause request nil)
|
||||
(handle-exception cause request nil)))
|
||||
|
||||
(defn handle'
|
||||
[cause request]
|
||||
(::rres/body (handle cause request)))
|
||||
(if (or (instance? java.util.concurrent.CompletionException cause)
|
||||
(instance? java.util.concurrent.ExecutionException cause))
|
||||
(handle-exception (ex-cause cause) request)
|
||||
(handle-exception cause request)))
|
||||
|
||||
@@ -12,17 +12,19 @@
|
||||
[app.config :as cf]
|
||||
[app.util.json :as json]
|
||||
[cuerdas.core :as str]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as rres]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]
|
||||
[promesa.util :as pu]
|
||||
[yetti.adapter :as yt]
|
||||
[yetti.middleware :as ymw])
|
||||
[yetti.middleware :as ymw]
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as yrs])
|
||||
(:import
|
||||
com.fasterxml.jackson.core.JsonParseException
|
||||
com.fasterxml.jackson.core.io.JsonEOFException
|
||||
com.fasterxml.jackson.databind.exc.MismatchedInputException
|
||||
io.undertow.server.RequestTooBigException
|
||||
java.io.InputStream
|
||||
java.io.OutputStream))
|
||||
java.io.OutputStream
|
||||
java.io.InputStream))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
@@ -43,17 +45,17 @@
|
||||
(defn wrap-parse-request
|
||||
[handler]
|
||||
(letfn [(process-request [request]
|
||||
(let [header (rreq/get-header request "content-type")]
|
||||
(let [header (yrq/get-header request "content-type")]
|
||||
(cond
|
||||
(str/starts-with? header "application/transit+json")
|
||||
(with-open [^InputStream is (rreq/body request)]
|
||||
(with-open [^InputStream is (yrq/body request)]
|
||||
(let [params (t/read! (t/reader is))]
|
||||
(-> request
|
||||
(assoc :body-params params)
|
||||
(update :params merge params))))
|
||||
|
||||
(str/starts-with? header "application/json")
|
||||
(with-open [^InputStream is (rreq/body request)]
|
||||
(with-open [^InputStream is (yrq/body request)]
|
||||
(let [params (json/decode is json-mapper)]
|
||||
(-> request
|
||||
(assoc :body-params params)
|
||||
@@ -62,36 +64,35 @@
|
||||
:else
|
||||
request)))
|
||||
|
||||
(handle-error [cause]
|
||||
(handle-error [raise cause]
|
||||
(cond
|
||||
(instance? RuntimeException cause)
|
||||
(if-let [cause (ex-cause cause)]
|
||||
(handle-error cause)
|
||||
(throw cause))
|
||||
(handle-error raise cause)
|
||||
(raise cause))
|
||||
|
||||
(instance? RequestTooBigException cause)
|
||||
(ex/raise :type :validation
|
||||
:code :request-body-too-large
|
||||
:hint (ex-message cause))
|
||||
(raise (ex/error :type :validation
|
||||
:code :request-body-too-large
|
||||
:hint (ex-message cause)))
|
||||
|
||||
|
||||
(or (instance? JsonEOFException cause)
|
||||
(instance? JsonParseException cause)
|
||||
(instance? MismatchedInputException cause))
|
||||
(ex/raise :type :validation
|
||||
:code :malformed-json
|
||||
:hint (ex-message cause)
|
||||
:cause cause)
|
||||
|
||||
(instance? JsonParseException cause))
|
||||
(raise (ex/error :type :validation
|
||||
:code :malformed-json
|
||||
:hint (ex-message cause)
|
||||
:cause cause))
|
||||
:else
|
||||
(throw cause)))]
|
||||
(raise cause)))]
|
||||
|
||||
(fn [request]
|
||||
(if (= (rreq/method request) :post)
|
||||
(fn [request respond raise]
|
||||
(if (= (yrq/method request) :post)
|
||||
(let [request (ex/try! (process-request request))]
|
||||
(if (ex/exception? request)
|
||||
(handle-error request)
|
||||
(handler request)))
|
||||
(handler request)))))
|
||||
(handle-error raise request)
|
||||
(handler request respond raise)))
|
||||
(handler request respond raise)))))
|
||||
|
||||
(def parse-request
|
||||
{:name ::parse-request
|
||||
@@ -109,7 +110,7 @@
|
||||
(defn wrap-format-response
|
||||
[handler]
|
||||
(letfn [(transit-streamable-body [data opts]
|
||||
(reify rres/StreamableResponseBody
|
||||
(reify yrs/StreamableResponseBody
|
||||
(-write-body-to-stream [_ _ output-stream]
|
||||
(try
|
||||
(with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
|
||||
@@ -117,14 +118,13 @@
|
||||
(t/write! tw data)))
|
||||
(catch java.io.IOException _)
|
||||
(catch Throwable cause
|
||||
(binding [l/*context* {:value data}]
|
||||
(l/error :hint "unexpected error on encoding response"
|
||||
:cause cause)))
|
||||
(l/warn :hint "unexpected error on encoding response"
|
||||
:cause cause))
|
||||
(finally
|
||||
(.close ^OutputStream output-stream))))))
|
||||
|
||||
(json-streamable-body [data]
|
||||
(reify rres/StreamableResponseBody
|
||||
(reify yrs/StreamableResponseBody
|
||||
(-write-body-to-stream [_ _ output-stream]
|
||||
(try
|
||||
(with-open [^OutputStream bos (buffered-output-stream output-stream buffer-size)]
|
||||
@@ -132,31 +132,30 @@
|
||||
|
||||
(catch java.io.IOException _)
|
||||
(catch Throwable cause
|
||||
(binding [l/*context* {:value data}]
|
||||
(l/error :hint "unexpected error on encoding response"
|
||||
:cause cause)))
|
||||
(l/warn :hint "unexpected error on encoding response"
|
||||
:cause cause))
|
||||
(finally
|
||||
(.close ^OutputStream output-stream))))))
|
||||
|
||||
(format-response-with-json [response _]
|
||||
(let [body (::rres/body response)]
|
||||
(let [body (::yrs/body response)]
|
||||
(if (or (boolean? body) (coll? body))
|
||||
(-> response
|
||||
(update ::rres/headers assoc "content-type" "application/json")
|
||||
(assoc ::rres/body (json-streamable-body body)))
|
||||
(update ::yrs/headers assoc "content-type" "application/json")
|
||||
(assoc ::yrs/body (json-streamable-body body)))
|
||||
response)))
|
||||
|
||||
(format-response-with-transit [response request]
|
||||
(let [body (::rres/body response)]
|
||||
(let [body (::yrs/body response)]
|
||||
(if (or (boolean? body) (coll? body))
|
||||
(let [qs (rreq/query request)
|
||||
(let [qs (yrq/query request)
|
||||
opts (if (or (contains? cf/flags :transit-readable-response)
|
||||
(str/includes? qs "transit_verbose"))
|
||||
{:type :json-verbose}
|
||||
{:type :json})]
|
||||
(-> response
|
||||
(update ::rres/headers assoc "content-type" "application/transit+json")
|
||||
(assoc ::rres/body (transit-streamable-body body opts))))
|
||||
(update ::yrs/headers assoc "content-type" "application/transit+json")
|
||||
(assoc ::yrs/body (transit-streamable-body body opts))))
|
||||
response)))
|
||||
|
||||
(format-from-params [{:keys [query-params] :as request}]
|
||||
@@ -165,7 +164,7 @@
|
||||
|
||||
(format-response [response request]
|
||||
(let [accept (or (format-from-params request)
|
||||
(rreq/get-header request "accept"))]
|
||||
(yrq/get-header request "accept"))]
|
||||
(cond
|
||||
(or (= accept "application/transit+json")
|
||||
(str/includes? accept "application/transit+json"))
|
||||
@@ -182,9 +181,11 @@
|
||||
(cond-> response
|
||||
(map? response) (format-response request)))]
|
||||
|
||||
(fn [request]
|
||||
(let [response (handler request)]
|
||||
(process-response response request)))))
|
||||
(fn [request respond raise]
|
||||
(handler request
|
||||
(fn [response]
|
||||
(respond (process-response response request)))
|
||||
raise))))
|
||||
|
||||
(def format-response
|
||||
{:name ::format-response
|
||||
@@ -192,11 +193,12 @@
|
||||
|
||||
(defn wrap-errors
|
||||
[handler on-error]
|
||||
(fn [request]
|
||||
(try
|
||||
(handler request)
|
||||
(catch Throwable cause
|
||||
(on-error cause request)))))
|
||||
(fn [request respond raise]
|
||||
(handler request respond (fn [cause]
|
||||
(try
|
||||
(respond (on-error cause request))
|
||||
(catch Throwable cause
|
||||
(raise cause)))))))
|
||||
|
||||
(def errors
|
||||
{:name ::errors
|
||||
@@ -214,11 +216,11 @@
|
||||
(defn wrap-cors
|
||||
[handler]
|
||||
(fn [request]
|
||||
(let [response (if (= (rreq/method request) :options)
|
||||
{::rres/status 200}
|
||||
(let [response (if (= (yrq/method request) :options)
|
||||
{::yrs/status 200}
|
||||
(handler request))
|
||||
origin (rreq/get-header request "origin")]
|
||||
(update response ::rres/headers with-cors-headers origin))))
|
||||
origin (yrq/get-header request "origin")]
|
||||
(update response ::yrs/headers with-cors-headers origin))))
|
||||
|
||||
(def cors
|
||||
{:name ::cors
|
||||
@@ -232,8 +234,18 @@
|
||||
(fn [data _]
|
||||
(when-let [allowed (:allowed-methods data)]
|
||||
(fn [handler]
|
||||
(fn [request]
|
||||
(let [method (rreq/method request)]
|
||||
(fn [request respond raise]
|
||||
(let [method (yrq/method request)]
|
||||
(if (contains? allowed method)
|
||||
(handler request)
|
||||
{::rres/status 405}))))))})
|
||||
(handler request respond raise)
|
||||
(respond {::yrs/status 405})))))))})
|
||||
|
||||
(def with-dispatch
|
||||
{:name ::with-dispatch
|
||||
:compile
|
||||
(fn [& _]
|
||||
(fn [handler executor]
|
||||
(let [executor (px/resolve-executor executor)]
|
||||
(fn [request respond raise]
|
||||
(->> (px/submit! executor (partial handler request))
|
||||
(p/fnly (pu/handler respond raise)))))))})
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[ring.request :as rreq]
|
||||
[yetti.request :as yrq]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
@@ -143,7 +142,7 @@
|
||||
(us/assert! ::us/uuid profile-id)
|
||||
|
||||
(fn [request response]
|
||||
(let [uagent (rreq/get-header request "user-agent")
|
||||
(let [uagent (yrq/get-header request "user-agent")
|
||||
params {:profile-id profile-id
|
||||
:user-agent uagent
|
||||
:created-at (dt/now)}
|
||||
@@ -210,8 +209,9 @@
|
||||
(l/trace :hint "exception on decoding malformed token" :cause cause)
|
||||
request)))]
|
||||
|
||||
(fn [request]
|
||||
(handler (handle-request request)))))
|
||||
(fn [request respond raise]
|
||||
(let [request (handle-request request)]
|
||||
(handler request respond raise)))))
|
||||
|
||||
(defn- wrap-authz
|
||||
[handler {:keys [::manager]}]
|
||||
@@ -221,15 +221,12 @@
|
||||
request (cond-> request
|
||||
(some? session)
|
||||
(assoc ::profile-id (:profile-id session)
|
||||
::id (:id session)))
|
||||
response (handler request)]
|
||||
::id (:id session)))]
|
||||
|
||||
(if (renew-session? session)
|
||||
(let [session (update! manager session)]
|
||||
(-> response
|
||||
(assign-auth-token-cookie session)
|
||||
(assign-authenticated-cookie session)))
|
||||
response))))
|
||||
(cond-> (handler request)
|
||||
(renew-session? session)
|
||||
(-> (assign-auth-token-cookie session)
|
||||
(assign-authenticated-cookie session))))))
|
||||
|
||||
(def soft-auth
|
||||
{:name ::soft-auth
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.http.sse
|
||||
"SSE (server sent events) helpers"
|
||||
(:refer-clojure :exclude [tap])
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.http.errors :as errors]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.csp :as sp]
|
||||
[promesa.util :as pu]
|
||||
[ring.response :as rres])
|
||||
(:import
|
||||
java.io.OutputStream))
|
||||
|
||||
(def ^:dynamic *channel* nil)
|
||||
|
||||
(defn- write!
|
||||
[^OutputStream output ^bytes data]
|
||||
(l/trc :hint "writting data" :data data :length (alength data))
|
||||
(.write output data)
|
||||
(.flush output))
|
||||
|
||||
(defn- create-writer-loop
|
||||
[^OutputStream output]
|
||||
(try
|
||||
(loop []
|
||||
(when-let [event (sp/take! *channel*)]
|
||||
(let [result (ex/try! (write! output event))]
|
||||
(if (ex/exception? result)
|
||||
(l/wrn :hint "unexpected exception on sse writer" :cause result)
|
||||
(recur)))))
|
||||
(finally
|
||||
(pu/close! output))))
|
||||
|
||||
(defn- encode
|
||||
[[name data]]
|
||||
(try
|
||||
(let [data (with-out-str
|
||||
(println "event:" (d/name name))
|
||||
(println "data:" (t/encode-str data {:type :json-verbose}))
|
||||
(println))]
|
||||
(.getBytes data "UTF-8"))
|
||||
(catch Throwable cause
|
||||
(l/err :hint "unexpected error on encoding value on sse stream"
|
||||
:cause cause)
|
||||
nil)))
|
||||
|
||||
;; ---- PUBLIC API
|
||||
|
||||
(def default-headers
|
||||
{"Content-Type" "text/event-stream;charset=UTF-8"
|
||||
"Cache-Control" "no-cache, no-store, max-age=0, must-revalidate"
|
||||
"Pragma" "no-cache"})
|
||||
|
||||
(defn tap
|
||||
([data] (tap "event" data))
|
||||
([name data]
|
||||
(when-let [channel *channel*]
|
||||
(sp/put! channel [name data])
|
||||
nil)))
|
||||
|
||||
(defn response
|
||||
[handler & {:keys [buf] :or {buf 32} :as opts}]
|
||||
(fn [request]
|
||||
{::rres/headers default-headers
|
||||
::rres/status 200
|
||||
::rres/body (reify rres/StreamableResponseBody
|
||||
(-write-body-to-stream [_ _ output]
|
||||
(binding [*channel* (sp/chan :buf buf :xf (keep encode))]
|
||||
(let [writer (px/run! :virtual (partial create-writer-loop output))]
|
||||
(try
|
||||
(tap "end" (handler))
|
||||
(catch Throwable cause
|
||||
(tap "error" (errors/handle' cause request)))
|
||||
(finally
|
||||
(sp/close! *channel*)
|
||||
(p/await! writer)))))))}))
|
||||
@@ -10,8 +10,7 @@
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.common.spec :as us]
|
||||
[app.db :as db]
|
||||
[app.http.session :as session]
|
||||
[app.metrics :as mtx]
|
||||
@@ -21,7 +20,6 @@
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.exec.csp :as sp]
|
||||
[ring.websocket :as rws]
|
||||
[yetti.websocket :as yws]))
|
||||
|
||||
(def recv-labels
|
||||
@@ -101,10 +99,7 @@
|
||||
(sp/pipe ch output-ch false)
|
||||
|
||||
;; Subscribe to the profile topic on msgbus/redis
|
||||
(mbus/sub! msgbus :topic profile-id :chan ch)
|
||||
|
||||
;; Subscribe to the system topic on msgbus/redis
|
||||
(mbus/sub! msgbus :topic (str uuid/zero) :chan ch)))
|
||||
(mbus/sub! msgbus :topic profile-id :chan ch)))
|
||||
|
||||
(defmethod handle-message :close
|
||||
[{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::profile-id ::session-id]} _]
|
||||
@@ -117,21 +112,21 @@
|
||||
:profile-id profile-id
|
||||
:session-id session-id}]
|
||||
|
||||
;; Close profile subscription if exists
|
||||
(when-let [ch (:channel psub)]
|
||||
(sp/close! ch)
|
||||
(mbus/purge! msgbus [ch]))
|
||||
;; Close profile subscription if exists
|
||||
(when-let [ch (:channel psub)]
|
||||
(sp/close! ch)
|
||||
(mbus/purge! msgbus [ch]))
|
||||
|
||||
;; Close team subscription if exists
|
||||
(when-let [ch (:channel tsub)]
|
||||
(sp/close! ch)
|
||||
(mbus/purge! msgbus [ch]))
|
||||
;; Close team subscription if exists
|
||||
(when-let [ch (:channel tsub)]
|
||||
(sp/close! ch)
|
||||
(mbus/purge! msgbus [ch]))
|
||||
|
||||
;; Close file subscription if exists
|
||||
(when-let [{:keys [topic channel]} fsub]
|
||||
(sp/close! channel)
|
||||
(mbus/purge! msgbus [channel])
|
||||
(mbus/pub! msgbus :topic topic :message msg))))
|
||||
;; Close file subscription if exists
|
||||
(when-let [{:keys [topic channel]} fsub]
|
||||
(sp/close! channel)
|
||||
(mbus/purge! msgbus [channel])
|
||||
(mbus/pub! msgbus :topic topic :message msg))))
|
||||
|
||||
(defmethod handle-message :subscribe-team
|
||||
[{:keys [::mbus/msgbus]} {:keys [::ws/id ::ws/state ::ws/output-ch ::session-id]} {:keys [team-id] :as params}]
|
||||
@@ -179,7 +174,7 @@
|
||||
(let [message {:type :presence
|
||||
:file-id file-id
|
||||
:session-id session-id
|
||||
:profile-id profile-id}]
|
||||
:profile-id profile-id}]
|
||||
(mbus/pub! msgbus
|
||||
:topic file-id
|
||||
:message message)))
|
||||
@@ -278,23 +273,19 @@
|
||||
:inc 1)
|
||||
message)
|
||||
|
||||
(def ^:private schema:params
|
||||
(sm/define
|
||||
[:map {:title "params"}
|
||||
[:session-id ::sm/uuid]]))
|
||||
|
||||
(s/def ::session-id ::us/uuid)
|
||||
(s/def ::handler-params
|
||||
(s/keys :req-un [::session-id]))
|
||||
|
||||
(defn- http-handler
|
||||
[cfg {:keys [params ::session/profile-id] :as request}]
|
||||
(let [{:keys [session-id]} (sm/conform! schema:params params)]
|
||||
(let [{:keys [session-id]} (us/conform ::handler-params params)]
|
||||
(cond
|
||||
(not profile-id)
|
||||
(ex/raise :type :authentication
|
||||
:hint "Authentication required.")
|
||||
|
||||
;; WORKAROUND: we use the adapter specific predicate for
|
||||
;; performance reasons; for now, the ring default impl for
|
||||
;; `upgrade-request?` parses all requests headers before perform
|
||||
;; any checking.
|
||||
(not (yws/upgrade-request? request))
|
||||
(ex/raise :type :validation
|
||||
:code :websocket-request-expected
|
||||
@@ -303,13 +294,14 @@
|
||||
:else
|
||||
(do
|
||||
(l/trace :hint "websocket request" :profile-id profile-id :session-id session-id)
|
||||
{::rws/listener (ws/listener request
|
||||
::ws/on-rcv-message (partial on-rcv-message cfg)
|
||||
::ws/on-snd-message (partial on-snd-message cfg)
|
||||
::ws/on-connect (partial on-connect cfg)
|
||||
::ws/handler (partial handle-message cfg)
|
||||
::profile-id profile-id
|
||||
::session-id session-id)}))))
|
||||
(->> (ws/handler
|
||||
::ws/on-rcv-message (partial on-rcv-message cfg)
|
||||
::ws/on-snd-message (partial on-snd-message cfg)
|
||||
::ws/on-connect (partial on-connect cfg)
|
||||
::ws/handler (partial handle-message cfg)
|
||||
::profile-id profile-id
|
||||
::session-id session-id)
|
||||
(yws/upgrade request))))))
|
||||
|
||||
(defmethod ig/pre-init-spec ::routes [_]
|
||||
(s/keys :req [::mbus/msgbus
|
||||
@@ -322,4 +314,5 @@
|
||||
(defmethod ig/init-key ::routes
|
||||
[_ cfg]
|
||||
["/ws/notifications" {:middleware [[session/authz cfg]]
|
||||
:handler (partial http-handler cfg)}])
|
||||
:handler (partial http-handler cfg)
|
||||
:allowed-methods #{:get}}])
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
[integrant.core :as ig]
|
||||
[lambdaisland.uri :as u]
|
||||
[promesa.exec :as px]
|
||||
[ring.request :as rreq]))
|
||||
[yetti.request :as yrq]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HELPERS
|
||||
@@ -41,9 +41,9 @@
|
||||
|
||||
(defn parse-client-ip
|
||||
[request]
|
||||
(or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
|
||||
(rreq/get-header request "x-real-ip")
|
||||
(some-> (rreq/remote-addr request) str)))
|
||||
(or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
|
||||
(yrq/get-header request "x-real-ip")
|
||||
(some-> (yrq/remote-addr request) str)))
|
||||
|
||||
(defn extract-utm-params
|
||||
"Extracts additional data from params and namespace them under
|
||||
|
||||
@@ -39,40 +39,35 @@
|
||||
(defn record->report
|
||||
[{:keys [::l/context ::l/message ::l/props ::l/logger ::l/level ::l/cause] :as record}]
|
||||
(us/assert! ::l/record record)
|
||||
(if (or (instance? java.util.concurrent.CompletionException cause)
|
||||
(instance? java.util.concurrent.ExecutionException cause))
|
||||
(-> record
|
||||
(assoc ::trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false))
|
||||
(assoc ::l/cause (ex-cause cause))
|
||||
(record->report))
|
||||
|
||||
(let [data (ex-data cause)
|
||||
ctx (-> context
|
||||
(let [data (ex-data cause)]
|
||||
(merge
|
||||
{:context (-> context
|
||||
(assoc :tenant (cf/get :tenant))
|
||||
(assoc :host (cf/get :host))
|
||||
(assoc :public-uri (cf/get :public-uri))
|
||||
(assoc :logger/name logger)
|
||||
(assoc :logger/level level)
|
||||
(dissoc :request/params :value :params :data))]
|
||||
(merge
|
||||
{:context (-> (into (sorted-map) ctx)
|
||||
(pp/pprint-str :length 50))
|
||||
:props (pp/pprint-str props :length 50)
|
||||
:hint (or (ex-message cause) @message)
|
||||
:trace (or (::trace record)
|
||||
(ex/format-throwable cause :data? false :explain? false :header? false :summary? false))}
|
||||
(assoc :version (:full cf/version))
|
||||
(assoc :logger-name logger)
|
||||
(assoc :logger-level level)
|
||||
(dissoc :params)
|
||||
(pp/pprint-str :width 200))
|
||||
|
||||
(when-let [params (or (:request/params context) (:params context))]
|
||||
{:params (pp/pprint-str params :length 30 :level 12)})
|
||||
:props (pp/pprint-str props :width 200)
|
||||
:hint (or (ex-message cause) @message)
|
||||
:trace (ex/format-throwable cause :data? false :explain? false :header? false :summary? false)}
|
||||
|
||||
(when-let [value (:value context)]
|
||||
{:value (pp/pprint-str value :length 30 :level 12)})
|
||||
(when-let [params (:params context)]
|
||||
{:params (pp/pprint-str params :width 200)})
|
||||
|
||||
(when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
|
||||
{:data (pp/pprint-str data :length 30 :level 12)})
|
||||
(when-let [data (some-> data (dissoc ::s/problems ::s/value ::s/spec ::sm/explain :hint))]
|
||||
{:data (pp/pprint-str data :width 200)})
|
||||
|
||||
(when-let [value (-> data ::sm/explain :value)]
|
||||
{:value (pp/pprint-str value :width 200)})
|
||||
|
||||
(when-let [explain (ex/explain data)]
|
||||
{:explain explain}))))
|
||||
|
||||
(when-let [explain (ex/explain data :length 30 :level 12)]
|
||||
{:explain explain})))))
|
||||
|
||||
(defn error-record?
|
||||
[{:keys [::l/level ::l/cause]}]
|
||||
@@ -96,11 +91,11 @@
|
||||
|
||||
(defmethod ig/init-key ::reporter
|
||||
[_ cfg]
|
||||
(let [input (sp/chan :buf (sp/sliding-buffer 64)
|
||||
(let [input (sp/chan :buf (sp/sliding-buffer 32)
|
||||
:xf (filter error-record?))]
|
||||
(add-watch l/log-record ::reporter #(sp/put! input %4))
|
||||
|
||||
(px/thread {:name "penpot/database-reporter"}
|
||||
(px/thread {:name "penpot/database-reporter" :virtual true}
|
||||
(l/info :hint "initializing database error persistence")
|
||||
(try
|
||||
(loop []
|
||||
|
||||
@@ -30,9 +30,7 @@
|
||||
"```\n"
|
||||
"- host: `" (:host report) "`\n"
|
||||
"- tenant: `" (:tenant report) "`\n"
|
||||
"- request-path: `" (:request-path report) "`\n"
|
||||
"- frontend-version: `" (:frontend-version report) "`\n"
|
||||
"- backend-version: `" (:backend-version report) "`\n"
|
||||
"- version: `" (:version report) "`\n"
|
||||
"\n"
|
||||
"Trace:\n"
|
||||
(:trace report)
|
||||
@@ -52,15 +50,13 @@
|
||||
(defn record->report
|
||||
[{:keys [::l/context ::l/id ::l/cause] :as record}]
|
||||
(us/assert! ::l/record record)
|
||||
{:id id
|
||||
:tenant (cf/get :tenant)
|
||||
:host (cf/get :host)
|
||||
:public-uri (cf/get :public-uri)
|
||||
:backend-version (or (:version/backend context) (:full cf/version))
|
||||
:frontend-version (:version/frontend context)
|
||||
:profile-id (:request/profile-id context)
|
||||
:request-path (:request/path context)
|
||||
:trace (ex/format-throwable cause :detail? false :header? false)})
|
||||
{:id id
|
||||
:tenant (cf/get :tenant)
|
||||
:host (cf/get :host)
|
||||
:public-uri (cf/get :public-uri)
|
||||
:version (:full cf/version)
|
||||
:profile-id (:profile-id context)
|
||||
:trace (ex/format-throwable cause :detail? false :header? false)})
|
||||
|
||||
(defn handle-event
|
||||
[cfg record]
|
||||
|
||||
@@ -182,4 +182,5 @@
|
||||
"invalid-uri"
|
||||
|
||||
(instance? java.net.http.HttpConnectTimeoutException cause)
|
||||
"timeout"))
|
||||
"timeout"
|
||||
))
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
[app.auth.oidc :as-alias oidc]
|
||||
[app.auth.oidc.providers :as-alias oidc.providers]
|
||||
[app.common.logging :as l]
|
||||
[app.common.svg :as csvg]
|
||||
[app.config :as cf]
|
||||
[app.db :as-alias db]
|
||||
[app.email :as-alias email]
|
||||
@@ -30,19 +29,14 @@
|
||||
[app.redis :as-alias rds]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.doc :as-alias rpc.doc]
|
||||
[app.setup :as-alias setup]
|
||||
[app.srepl :as-alias srepl]
|
||||
[app.storage :as-alias sto]
|
||||
[app.storage.fs :as-alias sto.fs]
|
||||
[app.storage.s3 :as-alias sto.s3]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[cider.nrepl :refer [cider-nrepl-handler]]
|
||||
[clojure.test :as test]
|
||||
[clojure.tools.namespace.repl :as repl]
|
||||
[cuerdas.core :as str]
|
||||
[integrant.core :as ig]
|
||||
[nrepl.server :as nrepl]
|
||||
[promesa.exec :as px])
|
||||
(:gen-class))
|
||||
|
||||
@@ -160,6 +154,12 @@
|
||||
{::mdef/name "penpot_executors_running_threads"
|
||||
::mdef/help "Current number of threads with state RUNNING."
|
||||
::mdef/labels ["name"]
|
||||
::mdef/type :gauge}
|
||||
|
||||
:executors-queued-submissions
|
||||
{::mdef/name "penpot_executors_queued_submissions"
|
||||
::mdef/help "Current number of queued submissions."
|
||||
::mdef/labels ["name"]
|
||||
::mdef/type :gauge}})
|
||||
|
||||
(def system-config
|
||||
@@ -174,12 +174,13 @@
|
||||
|
||||
;; Default thread pool for IO operations
|
||||
::wrk/executor
|
||||
{}
|
||||
{::wrk/parallelism (cf/get :default-executor-parallelism
|
||||
(+ 3 (* (px/get-available-processors) 3)))}
|
||||
|
||||
::wrk/monitor
|
||||
{::mtx/metrics (ig/ref ::mtx/metrics)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
::wrk/name "default"}
|
||||
::wrk/name "default"
|
||||
::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
:app.migrations/migrations
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
@@ -210,7 +211,7 @@
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
::http.client/client
|
||||
{}
|
||||
{::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
::session/manager
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
@@ -219,14 +220,16 @@
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
|
||||
::http.awsns/routes
|
||||
{::props (ig/ref ::setup/props)
|
||||
{::props (ig/ref :app.setup/props)
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
::http.client/client (ig/ref ::http.client/client)
|
||||
::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
::http/server
|
||||
{::http/port (cf/get :http-server-port)
|
||||
::http/host (cf/get :http-server-host)
|
||||
::http/router (ig/ref ::http/router)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
::http/io-threads (cf/get :http-server-io-threads)
|
||||
::http/max-body-size (cf/get :http-server-max-body-size)
|
||||
::http/max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
|
||||
@@ -260,7 +263,7 @@
|
||||
::oidc/routes
|
||||
{::http.client/client (ig/ref ::http.client/client)
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::props (ig/ref ::setup/props)
|
||||
::props (ig/ref :app.setup/props)
|
||||
::oidc/providers {:google (ig/ref ::oidc.providers/google)
|
||||
:github (ig/ref ::oidc.providers/github)
|
||||
:gitlab (ig/ref ::oidc.providers/gitlab)
|
||||
@@ -272,7 +275,7 @@
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::rpc/routes (ig/ref ::rpc/routes)
|
||||
::rpc.doc/routes (ig/ref ::rpc.doc/routes)
|
||||
::props (ig/ref ::setup/props)
|
||||
::props (ig/ref :app.setup/props)
|
||||
::mtx/routes (ig/ref ::mtx/routes)
|
||||
::oidc/routes (ig/ref ::oidc/routes)
|
||||
::http.debug/routes (ig/ref ::http.debug/routes)
|
||||
@@ -280,11 +283,11 @@
|
||||
::http.ws/routes (ig/ref ::http.ws/routes)
|
||||
::http.awsns/routes (ig/ref ::http.awsns/routes)}
|
||||
|
||||
::http.debug/routes
|
||||
:app.http.debug/routes
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
::session/manager (ig/ref ::session/manager)
|
||||
::sto/storage (ig/ref ::sto/storage)
|
||||
::props (ig/ref ::setup/props)}
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
::http.ws/routes
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
@@ -296,10 +299,12 @@
|
||||
{::http.assets/path (cf/get :assets-path)
|
||||
::http.assets/cache-max-age (dt/duration {:hours 24})
|
||||
::http.assets/cache-max-agesignature-max-age (dt/duration {:hours 24 :minutes 5})
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
::sto/storage (ig/ref ::sto/storage)
|
||||
::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
:app.rpc/climit
|
||||
{::mtx/metrics (ig/ref ::mtx/metrics)}
|
||||
{::mtx/metrics (ig/ref ::mtx/metrics)
|
||||
::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
:app.rpc/rlimit
|
||||
{::wrk/executor (ig/ref ::wrk/executor)}
|
||||
@@ -314,14 +319,15 @@
|
||||
::mtx/metrics (ig/ref ::mtx/metrics)
|
||||
::mbus/msgbus (ig/ref ::mbus/msgbus)
|
||||
::rds/redis (ig/ref ::rds/redis)
|
||||
::csvg/optimizer (ig/ref ::csvg/optimizer)
|
||||
|
||||
::rpc/climit (ig/ref ::rpc/climit)
|
||||
::rpc/rlimit (ig/ref ::rpc/rlimit)
|
||||
::setup/templates (ig/ref ::setup/templates)
|
||||
::props (ig/ref ::setup/props)
|
||||
|
||||
:pool (ig/ref ::db/pool)}
|
||||
::props (ig/ref :app.setup/props)
|
||||
|
||||
:pool (ig/ref ::db/pool)
|
||||
:templates (ig/ref :app.setup/builtin-templates)
|
||||
}
|
||||
|
||||
:app.rpc.doc/routes
|
||||
{:methods (ig/ref :app.rpc/methods)}
|
||||
@@ -329,8 +335,9 @@
|
||||
:app.rpc/routes
|
||||
{::rpc/methods (ig/ref :app.rpc/methods)
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
::session/manager (ig/ref ::session/manager)
|
||||
::props (ig/ref ::setup/props)}
|
||||
::props (ig/ref :app.setup/props)}
|
||||
|
||||
::wrk/registry
|
||||
{::mtx/metrics (ig/ref ::mtx/metrics)
|
||||
@@ -383,7 +390,7 @@
|
||||
:app.tasks.telemetry/handler
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::http.client/client (ig/ref ::http.client/client)
|
||||
::props (ig/ref ::setup/props)}
|
||||
::props (ig/ref :app.setup/props)}
|
||||
|
||||
[::srepl/urepl ::srepl/server]
|
||||
{::srepl/port (cf/get :urepl-port 6062)
|
||||
@@ -393,9 +400,10 @@
|
||||
{::srepl/port (cf/get :prepl-port 6063)
|
||||
::srepl/host (cf/get :prepl-host "localhost")}
|
||||
|
||||
::setup/templates {}
|
||||
:app.setup/builtin-templates
|
||||
{::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
::setup/props
|
||||
:app.setup/props
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::key (cf/get :secret-key)
|
||||
|
||||
@@ -403,11 +411,8 @@
|
||||
;; module requires the migrations to run before initialize.
|
||||
::migrations (ig/ref :app.migrations/migrations)}
|
||||
|
||||
::csvg/optimizer
|
||||
{}
|
||||
|
||||
::audit.tasks/archive
|
||||
{::props (ig/ref ::setup/props)
|
||||
{::props (ig/ref :app.setup/props)
|
||||
::db/pool (ig/ref ::db/pool)
|
||||
::http.client/client (ig/ref ::http.client/client)}
|
||||
|
||||
@@ -430,18 +435,20 @@
|
||||
|
||||
::sto/storage
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::wrk/executor (ig/ref ::wrk/executor)
|
||||
::sto/backends
|
||||
{:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
|
||||
:assets-fs (ig/ref [::assets :app.storage.fs/backend])}}
|
||||
|
||||
[::assets :app.storage.s3/backend]
|
||||
{::sto.s3/region (cf/get :storage-assets-s3-region)
|
||||
::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
|
||||
::sto.s3/bucket (cf/get :storage-assets-s3-bucket)
|
||||
::sto.s3/io-threads (cf/get :storage-assets-s3-io-threads)}
|
||||
{::sto.s3/region (cf/get :storage-assets-s3-region)
|
||||
::sto.s3/endpoint (cf/get :storage-assets-s3-endpoint)
|
||||
::sto.s3/bucket (cf/get :storage-assets-s3-bucket)
|
||||
::wrk/executor (ig/ref ::wrk/executor)}
|
||||
|
||||
[::assets :app.storage.fs/backend]
|
||||
{::sto.fs/directory (cf/get :storage-assets-fs-directory)}})
|
||||
{::sto.fs/directory (cf/get :storage-assets-fs-directory)}
|
||||
})
|
||||
|
||||
|
||||
(def worker-config
|
||||
@@ -515,65 +522,22 @@
|
||||
(merge worker-config))
|
||||
(ig/prep)
|
||||
(ig/init))))
|
||||
(l/inf :hint "welcome to penpot"
|
||||
:flags (str/join "," (map name cf/flags))
|
||||
:worker? (contains? cf/flags :backend-worker)
|
||||
:version (:full cf/version)))
|
||||
(l/info :hint "welcome to penpot"
|
||||
:flags (str/join "," (map name cf/flags))
|
||||
:worker? (contains? cf/flags :backend-worker)
|
||||
:version (:full cf/version)))
|
||||
|
||||
(defn stop
|
||||
[]
|
||||
(alter-var-root #'system (fn [sys]
|
||||
(when sys (ig/halt! sys))
|
||||
nil)))
|
||||
(defn restart
|
||||
[]
|
||||
(stop)
|
||||
(repl/refresh :after 'app.main/start))
|
||||
|
||||
(defn restart-all
|
||||
[]
|
||||
(stop)
|
||||
(repl/refresh-all :after 'app.main/start))
|
||||
|
||||
(defmacro run-bench
|
||||
[& exprs]
|
||||
`(do
|
||||
(require 'criterium.core)
|
||||
(criterium.core/with-progress-reporting (crit/quick-bench (do ~@exprs) :verbose))))
|
||||
|
||||
(defn run-tests
|
||||
([] (run-tests #"^backend-tests.*-test$"))
|
||||
([o]
|
||||
(repl/refresh)
|
||||
(cond
|
||||
(instance? java.util.regex.Pattern o)
|
||||
(test/run-all-tests o)
|
||||
|
||||
(symbol? o)
|
||||
(if-let [sns (namespace o)]
|
||||
(do (require (symbol sns))
|
||||
(test/test-vars [(resolve o)]))
|
||||
(test/test-ns o)))))
|
||||
|
||||
(repl/disable-reload! (find-ns 'integrant.core))
|
||||
|
||||
(defn -main
|
||||
[& _args]
|
||||
(try
|
||||
(let [p (promise)]
|
||||
(when (contains? cf/flags :nrepl-server)
|
||||
(l/inf :hint "start nrepl server" :port 6064)
|
||||
(nrepl/start-server :bind "0.0.0.0" :port 6064 :handler cider-nrepl-handler))
|
||||
|
||||
(start)
|
||||
(deref p))
|
||||
(start)
|
||||
(catch Throwable cause
|
||||
(binding [*out* *err*]
|
||||
(println "==== ERROR ===="))
|
||||
(.printStackTrace cause)
|
||||
(when-let [cause' (ex-cause cause)]
|
||||
(binding [*out* *err*]
|
||||
(println "==== CAUSE ===="))
|
||||
(.printStackTrace cause'))
|
||||
(px/sleep 500)
|
||||
(l/error :hint (ex-message cause)
|
||||
:cause cause)
|
||||
(System/exit -1))))
|
||||
|
||||
@@ -14,11 +14,11 @@
|
||||
[app.common.schema.generators :as sg]
|
||||
[app.common.schema.openapi :as-alias oapi]
|
||||
[app.common.spec :as us]
|
||||
[app.common.svg :as csvg]
|
||||
[app.config :as cf]
|
||||
[app.db :as-alias db]
|
||||
[app.storage :as-alias sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.svg :as svg]
|
||||
[app.util.time :as dt]
|
||||
[buddy.core.bytes :as bb]
|
||||
[buddy.core.codecs :as bc]
|
||||
@@ -201,7 +201,7 @@
|
||||
(us/assert ::input input)
|
||||
(let [{:keys [path mtype]} input]
|
||||
(if (= mtype "image/svg+xml")
|
||||
(let [info (some-> path slurp csvg/parse get-basic-info-from-svg)]
|
||||
(let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
|
||||
(when-not info
|
||||
(ex/raise :type :validation
|
||||
:code :invalid-svg-file
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
writer (StringWriter.)]
|
||||
(TextFormat/write004 writer samples)
|
||||
{:headers {"content-type" TextFormat/CONTENT_TYPE_004}
|
||||
:body (.toString writer)}))
|
||||
:body (.toString writer)}))
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -324,20 +324,7 @@
|
||||
{:name "0104-mod-file-thumbnail-table"
|
||||
:fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")}
|
||||
|
||||
{:name "0105-mod-file-change-table"
|
||||
:fn (mg/resource "app/migrations/sql/0105-mod-file-change-table.sql")}
|
||||
|
||||
{:name "0105-mod-server-error-report-table"
|
||||
:fn (mg/resource "app/migrations/sql/0105-mod-server-error-report-table.sql")}
|
||||
|
||||
{:name "0106-add-file-tagged-object-thumbnail-table"
|
||||
:fn (mg/resource "app/migrations/sql/0106-add-file-tagged-object-thumbnail-table.sql")}
|
||||
|
||||
{:name "0106-mod-team-table"
|
||||
:fn (mg/resource "app/migrations/sql/0106-mod-team-table.sql")}
|
||||
|
||||
{:name "0107-mod-file-tagged-object-thumbnail-table"
|
||||
:fn (mg/resource "app/migrations/sql/0107-mod-file-tagged-object-thumbnail-table.sql")}])
|
||||
])
|
||||
|
||||
(defn apply-migrations!
|
||||
[pool name migrations]
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
ALTER TABLE file_change
|
||||
ADD COLUMN label text NULL;
|
||||
|
||||
ALTER TABLE file_change
|
||||
ALTER COLUMN label SET STORAGE external;
|
||||
|
||||
CREATE INDEX file_change__label__idx
|
||||
ON file_change (file_id, label)
|
||||
WHERE label is not null;
|
||||
@@ -1,2 +0,0 @@
|
||||
CREATE INDEX server_error_report__created_at__idx
|
||||
ON server_error_report ( created_at );
|
||||
@@ -1,10 +0,0 @@
|
||||
CREATE TABLE file_tagged_object_thumbnail (
|
||||
file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
|
||||
tag text DEFAULT 'frame',
|
||||
object_id text NOT NULL,
|
||||
|
||||
media_id uuid NOT NULL REFERENCES storage_object(id) ON DELETE CASCADE DEFERRABLE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
|
||||
PRIMARY KEY(file_id, tag, object_id)
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE team ADD COLUMN features text[] NULL DEFAULT null;
|
||||
@@ -1,2 +0,0 @@
|
||||
CREATE INDEX file_tagged_object_thumbnail__media_id__idx
|
||||
ON file_tagged_object_thumbnail (media_id);
|
||||
@@ -30,11 +30,12 @@
|
||||
[app.storage :as-alias sto]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[integrant.core :as ig]
|
||||
[promesa.core :as p]
|
||||
[ring.request :as rreq]
|
||||
[ring.response :as rres]))
|
||||
[yetti.request :as yrq]
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
(s/def ::profile-id ::us/uuid)
|
||||
|
||||
@@ -57,23 +58,21 @@
|
||||
|
||||
(defn- handle-response
|
||||
[request result]
|
||||
(let [mdata (meta result)
|
||||
response (if (fn? result)
|
||||
(result request)
|
||||
(let [result (rph/unwrap result)]
|
||||
{::rres/status (::http/status mdata 200)
|
||||
::rres/headers (::http/headers mdata {})
|
||||
::rres/body result}))]
|
||||
(-> response
|
||||
(handle-response-transformation request mdata)
|
||||
(handle-before-comple-hook mdata))))
|
||||
(if (fn? result)
|
||||
(result request)
|
||||
(let [mdata (meta result)]
|
||||
(-> {::yrs/status (::http/status mdata 200)
|
||||
::yrs/headers (::http/headers mdata {})
|
||||
::yrs/body (rph/unwrap result)}
|
||||
(handle-response-transformation request mdata)
|
||||
(handle-before-comple-hook mdata)))))
|
||||
|
||||
(defn- rpc-handler
|
||||
"Ring handler that dispatches cmd requests and convert between
|
||||
internal async flow into ring async flow."
|
||||
[methods {:keys [params path-params] :as request}]
|
||||
(let [type (keyword (:type path-params))
|
||||
etag (rreq/get-header request "if-none-match")
|
||||
etag (yrq/get-header request "if-none-match")
|
||||
profile-id (or (::session/profile-id request)
|
||||
(::actoken/profile-id request))
|
||||
|
||||
@@ -139,20 +138,17 @@
|
||||
(f cfg (us/conform spec params)))
|
||||
f)))
|
||||
|
||||
;; TODO: integrate with sm/define
|
||||
|
||||
(defn- wrap-params-validation
|
||||
[_ f mdata]
|
||||
(if-let [schema (::sm/params mdata)]
|
||||
(let [schema (if (sm/lazy-schema? schema)
|
||||
schema
|
||||
(sm/define schema))
|
||||
validate (sm/validator schema)
|
||||
explain (sm/explainer schema)
|
||||
decode (sm/decoder schema)]
|
||||
(let [schema (sm/schema schema)
|
||||
valid? (sm/validator schema)
|
||||
explain (sm/explainer schema)
|
||||
decode (sm/decoder schema sm/default-transformer)]
|
||||
|
||||
(fn [cfg params]
|
||||
(let [params (decode params)]
|
||||
(if (validate params)
|
||||
(if (valid? params)
|
||||
(f cfg params)
|
||||
(ex/raise :type :validation
|
||||
:code :params-validation
|
||||
@@ -163,15 +159,13 @@
|
||||
[_ f mdata]
|
||||
(if (contains? cf/flags :rpc-output-validation)
|
||||
(or (when-let [schema (::sm/result mdata)]
|
||||
(let [schema (if (sm/lazy-schema? schema)
|
||||
schema
|
||||
(sm/define schema))
|
||||
validate (sm/validator schema)
|
||||
explain (sm/explainer schema)]
|
||||
(let [schema (sm/schema schema)
|
||||
valid? (sm/validator schema)
|
||||
explain (sm/explainer schema)]
|
||||
(fn [cfg params]
|
||||
(let [response (f cfg params)]
|
||||
(when (map? response)
|
||||
(when-not (validate response)
|
||||
(when-not (valid? response)
|
||||
(ex/raise :type :validation
|
||||
:code :data-validation
|
||||
::sm/explain (explain response))))
|
||||
@@ -220,7 +214,6 @@
|
||||
'app.rpc.commands.files-share
|
||||
'app.rpc.commands.files-temp
|
||||
'app.rpc.commands.files-update
|
||||
'app.rpc.commands.files-snapshot
|
||||
'app.rpc.commands.files-thumbnails
|
||||
'app.rpc.commands.ldap
|
||||
'app.rpc.commands.management
|
||||
@@ -243,7 +236,8 @@
|
||||
::ldap/provider
|
||||
::sto/storage
|
||||
::mtx/metrics
|
||||
::main/props]
|
||||
::main/props
|
||||
::wrk/executor]
|
||||
:opt [::climit
|
||||
::rlimit]
|
||||
:req-un [::db/pool]))
|
||||
@@ -262,6 +256,7 @@
|
||||
(s/keys :req [::methods
|
||||
::db/pool
|
||||
::main/props
|
||||
::wrk/executor
|
||||
::session/manager]))
|
||||
|
||||
(defmethod ig/init-key ::routes
|
||||
|
||||
@@ -31,24 +31,19 @@
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- id->str
|
||||
[id]
|
||||
(-> (str id)
|
||||
(subs 1)))
|
||||
|
||||
(defn- create-bulkhead-cache
|
||||
[config]
|
||||
(letfn [(load-fn [[id skey]]
|
||||
(when-let [config (get config id)]
|
||||
(l/trc :hint "insert into cache" :id (id->str id) :key skey)
|
||||
[{:keys [::wrk/executor]} config]
|
||||
(letfn [(load-fn [key]
|
||||
(let [config (get config (nth key 0))]
|
||||
(l/trace :hint "insert into cache" :key key)
|
||||
(pbh/create :permits (or (:permits config) (:concurrency config))
|
||||
:queue (or (:queue config) (:queue-size config))
|
||||
:timeout (:timeout config)
|
||||
:type :semaphore)))
|
||||
:executor executor
|
||||
:type (:type config :semaphore))))
|
||||
|
||||
(on-remove [key _ cause]
|
||||
(let [[id skey] key]
|
||||
(l/trc :hint "evict from cache" :id (id->str id) :key skey :reason (str cause))))]
|
||||
(on-remove [_ _ cause]
|
||||
(l/trace :hint "evict from cache" :key key :reason (str cause)))]
|
||||
|
||||
(cache/create :executor :same-thread
|
||||
:on-remove on-remove
|
||||
@@ -70,21 +65,22 @@
|
||||
|
||||
(s/def ::path ::fs/path)
|
||||
(defmethod ig/pre-init-spec ::rpc/climit [_]
|
||||
(s/keys :req [::mtx/metrics ::path]))
|
||||
(s/keys :req [::wrk/executor ::mtx/metrics ::path]))
|
||||
|
||||
(defmethod ig/init-key ::rpc/climit
|
||||
[_ {:keys [::path ::mtx/metrics] :as cfg}]
|
||||
[_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}]
|
||||
(when (contains? cf/flags :rpc-climit)
|
||||
(when-let [params (some->> path slurp edn/read-string)]
|
||||
(l/inf :hint "initializing concurrency limit" :config (str path))
|
||||
(l/info :hint "initializing concurrency limit" :config (str path))
|
||||
(us/verify! ::config params)
|
||||
{::cache (create-bulkhead-cache params)
|
||||
{::cache (create-bulkhead-cache cfg params)
|
||||
::config params
|
||||
::wrk/executor executor
|
||||
::mtx/metrics metrics})))
|
||||
|
||||
(s/def ::cache cache/cache?)
|
||||
(s/def ::instance
|
||||
(s/keys :req [::cache ::config]))
|
||||
(s/keys :req [::cache ::config ::wrk/executor]))
|
||||
|
||||
(s/def ::rpc/climit
|
||||
(s/nilable ::instance))
|
||||
@@ -95,92 +91,114 @@
|
||||
|
||||
(defn invoke!
|
||||
[cache metrics id key f]
|
||||
(if-let [limiter (cache/get cache [id key])]
|
||||
(let [tpoint (dt/tpoint)
|
||||
labels (into-array String [(id->str id)])
|
||||
wrapped (fn []
|
||||
(let [elapsed (tpoint)
|
||||
stats (pbh/get-stats limiter)]
|
||||
(l/trc :hint "acquired"
|
||||
:id (id->str id)
|
||||
:key key
|
||||
:permits (:permits stats)
|
||||
:queue (:queue stats)
|
||||
:max-permits (:max-permits stats)
|
||||
:max-queue (:max-queue stats)
|
||||
:elapsed (dt/format-duration elapsed))
|
||||
(let [limiter (cache/get cache [id key])
|
||||
tpoint (dt/tpoint)
|
||||
labels (into-array String [(name id)])
|
||||
|
||||
(mtx/run! metrics
|
||||
:id :rpc-climit-timing
|
||||
:val (inst-ms elapsed)
|
||||
:labels labels)
|
||||
(try
|
||||
(f)
|
||||
(finally
|
||||
(let [elapsed (tpoint)]
|
||||
(l/trc :hint "finished"
|
||||
:id (id->str id)
|
||||
:key key
|
||||
:permits (:permits stats)
|
||||
:queue (:queue stats)
|
||||
:max-permits (:max-permits stats)
|
||||
:max-queue (:max-queue stats)
|
||||
:elapsed (dt/format-duration elapsed)))))))
|
||||
measure!
|
||||
(fn [stats]
|
||||
wrapped
|
||||
(fn []
|
||||
(let [elapsed (tpoint)
|
||||
stats (pbh/get-stats limiter)]
|
||||
(l/trace :hint "executed"
|
||||
:id (name id)
|
||||
:key key
|
||||
:fnh (hash f)
|
||||
:permits (:permits stats)
|
||||
:queue (:queue stats)
|
||||
:max-permits (:max-permits stats)
|
||||
:max-queue (:max-queue stats)
|
||||
:elapsed (dt/format-duration elapsed))
|
||||
(mtx/run! metrics
|
||||
:id :rpc-climit-queue
|
||||
:val (:queue stats)
|
||||
:id :rpc-climit-timing
|
||||
:val (inst-ms elapsed)
|
||||
:labels labels)
|
||||
(mtx/run! metrics
|
||||
:id :rpc-climit-permits
|
||||
:val (:permits stats)
|
||||
:labels labels))]
|
||||
(try
|
||||
(f)
|
||||
(finally
|
||||
(let [elapsed (tpoint)]
|
||||
(l/trace :hint "finished"
|
||||
:id (name id)
|
||||
:key key
|
||||
:fnh (hash f)
|
||||
:permits (:permits stats)
|
||||
:queue (:queue stats)
|
||||
:max-permits (:max-permits stats)
|
||||
:max-queue (:max-queue stats)
|
||||
:elapsed (dt/format-duration elapsed)))))))
|
||||
measure!
|
||||
(fn [stats]
|
||||
(mtx/run! metrics
|
||||
:id :rpc-climit-queue
|
||||
:val (:queue stats)
|
||||
:labels labels)
|
||||
(mtx/run! metrics
|
||||
:id :rpc-climit-permits
|
||||
:val (:permits stats)
|
||||
:labels labels))]
|
||||
|
||||
(try
|
||||
(let [stats (pbh/get-stats limiter)]
|
||||
(measure! stats)
|
||||
(l/trc :hint "enqueued"
|
||||
:id (id->str id)
|
||||
(try
|
||||
(let [stats (pbh/get-stats limiter)]
|
||||
(measure! stats)
|
||||
(l/trace :hint "enqueued"
|
||||
:id (name id)
|
||||
:key key
|
||||
:fnh (hash f)
|
||||
:permits (:permits stats)
|
||||
:queue (:queue stats)
|
||||
:max-permits (:max-permits stats)
|
||||
:max-queue (:max-queue stats))
|
||||
(pbh/invoke! limiter wrapped))
|
||||
(catch ExceptionInfo cause
|
||||
(let [{:keys [type code]} (ex-data cause)]
|
||||
(if (= :bulkhead-error type)
|
||||
(ex/raise :type :concurrency-limit
|
||||
:code code
|
||||
:hint "concurrency limit reached")
|
||||
(throw cause))))
|
||||
(pbh/invoke! limiter wrapped))
|
||||
(catch ExceptionInfo cause
|
||||
(let [{:keys [type code]} (ex-data cause)]
|
||||
(if (= :bulkhead-error type)
|
||||
(ex/raise :type :concurrency-limit
|
||||
:code code
|
||||
:hint "concurrency limit reached")
|
||||
(throw cause))))
|
||||
|
||||
(finally
|
||||
(measure! (pbh/get-stats limiter)))))
|
||||
(finally
|
||||
(measure! (pbh/get-stats limiter))))))
|
||||
|
||||
(do
|
||||
(l/wrn :hint "unable to load limiter" :id (id->str id))
|
||||
(f))))
|
||||
|
||||
(defn configure
|
||||
[{:keys [::rpc/climit]} id]
|
||||
(us/assert! ::rpc/climit climit)
|
||||
(assoc climit ::id id))
|
||||
|
||||
(defn run!
|
||||
"Run a function in context of climit.
|
||||
Intended to be used in virtual threads."
|
||||
([{:keys [::id ::cache ::mtx/metrics]} f]
|
||||
(if (and cache id)
|
||||
(invoke! cache metrics id nil f)
|
||||
(f)))
|
||||
[{:keys [::id ::cache ::mtx/metrics]} f]
|
||||
(if (and cache id)
|
||||
(invoke! cache metrics id nil f)
|
||||
(f)))
|
||||
|
||||
([{:keys [::id ::cache ::mtx/metrics]} f executor]
|
||||
(let [f #(p/await! (px/submit! executor f))]
|
||||
(if (and cache id)
|
||||
(invoke! cache metrics id nil f)
|
||||
(f)))))
|
||||
(defn submit!
|
||||
[{:keys [::id ::cache ::wrk/executor ::mtx/metrics]} f]
|
||||
(let [f (partial px/submit! executor (px/wrap-bindings f))]
|
||||
(if (and cache id)
|
||||
(p/await! (invoke! cache metrics id nil f))
|
||||
(p/await! (f)))))
|
||||
|
||||
(defn configure
|
||||
([{:keys [::rpc/climit]} id]
|
||||
(us/assert! ::rpc/climit climit)
|
||||
(assoc climit ::id id))
|
||||
([{:keys [::rpc/climit]} id executor]
|
||||
(us/assert! ::rpc/climit climit)
|
||||
(-> climit
|
||||
(assoc ::id id)
|
||||
(assoc ::wrk/executor executor))))
|
||||
|
||||
(defmacro with-dispatch!
|
||||
"Dispatch blocking operation to a separated thread protected with the
|
||||
specified concurrency limiter. If climit is not active, the function
|
||||
will be scheduled to execute without concurrency monitoring."
|
||||
[instance & body]
|
||||
(if (vector? instance)
|
||||
`(-> (app.rpc.climit/configure ~@instance)
|
||||
(app.rpc.climit/run! (^:once fn* [] ~@body)))
|
||||
`(run! ~instance (^:once fn* [] ~@body))))
|
||||
|
||||
(defmacro with-dispatch
|
||||
"Dispatch blocking operation to a separated thread protected with
|
||||
the specified semaphore.
|
||||
DEPRECATED"
|
||||
[& params]
|
||||
`(with-dispatch! ~@params))
|
||||
|
||||
(def noop-fn (constantly nil))
|
||||
|
||||
@@ -189,19 +207,18 @@
|
||||
(if (and (some? climit) (some? id))
|
||||
(if-let [config (get-in climit [::config id])]
|
||||
(let [cache (::cache climit)]
|
||||
(l/dbg :hint "instrumenting method"
|
||||
:limit (id->str id)
|
||||
:service-name (::sv/name mdata)
|
||||
:timeout (:timeout config)
|
||||
:permits (:permits config)
|
||||
:queue (:queue config)
|
||||
:keyed? (not= key-fn noop-fn))
|
||||
|
||||
(l/debug :hint "wrap: instrumenting method"
|
||||
:limit (name id)
|
||||
:service-name (::sv/name mdata)
|
||||
:timeout (:timeout config)
|
||||
:permits (:permits config)
|
||||
:queue (:queue config)
|
||||
:keyed? (some? key-fn))
|
||||
(fn [cfg params]
|
||||
(invoke! cache metrics id (key-fn params) (partial f cfg params))))
|
||||
|
||||
(do
|
||||
(l/wrn :hint "no config found for specified queue" :id (id->str id))
|
||||
(l/warn :hint "no config found for specified queue" :id id)
|
||||
f))
|
||||
|
||||
f))
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
@@ -19,7 +19,9 @@
|
||||
[app.rpc.climit :as-alias climit]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.util.services :as sv]))
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
(defn- event->row [event]
|
||||
[(uuid/next)
|
||||
@@ -50,25 +52,26 @@
|
||||
(when (seq events)
|
||||
(db/insert-multi! pool :audit-log event-columns events))))
|
||||
|
||||
(def schema:event
|
||||
[:map {:title "Event"}
|
||||
[:name [:string {:max 250}]]
|
||||
[:type [:string {:max 250}]]
|
||||
[:props
|
||||
[:map-of :keyword :any]]
|
||||
[:context {:optional true}
|
||||
[:map-of :keyword :any]]])
|
||||
(s/def ::name ::us/string)
|
||||
(s/def ::type ::us/string)
|
||||
(s/def ::props (s/map-of ::us/keyword any?))
|
||||
(s/def ::timestamp dt/instant?)
|
||||
(s/def ::context (s/map-of ::us/keyword any?))
|
||||
|
||||
(def schema:push-audit-events
|
||||
[:map {:title "push-audit-events"}
|
||||
[:events [:vector schema:event]]])
|
||||
(s/def ::event
|
||||
(s/keys :req-un [::type ::name ::props ::timestamp]
|
||||
:opt-un [::context]))
|
||||
|
||||
(s/def ::events (s/every ::event))
|
||||
|
||||
(s/def ::push-audit-events
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::events]))
|
||||
|
||||
(sv/defmethod ::push-audit-events
|
||||
{::climit/id :submit-audit-events/by-profile
|
||||
{::climit/id :submit-audit-events-by-profile
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
::sm/params schema:push-audit-events
|
||||
::audit/skip true
|
||||
::doc/skip true
|
||||
::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} params]
|
||||
(if (or (db/read-only? pool)
|
||||
|
||||
@@ -6,13 +6,10 @@
|
||||
|
||||
(ns app.rpc.commands.auth
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
@@ -28,13 +25,31 @@
|
||||
[app.tokens :as tokens]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]))
|
||||
|
||||
(def schema:password
|
||||
[::sm/word-string {:max 500}])
|
||||
(s/def ::email ::us/email)
|
||||
(s/def ::fullname ::us/not-empty-string)
|
||||
(s/def ::lang ::us/string)
|
||||
(s/def ::path ::us/string)
|
||||
(s/def ::password ::us/not-empty-string)
|
||||
(s/def ::old-password ::us/not-empty-string)
|
||||
(s/def ::theme ::us/string)
|
||||
(s/def ::invitation-token ::us/not-empty-string)
|
||||
(s/def ::token ::us/not-empty-string)
|
||||
|
||||
(def schema:token
|
||||
[::sm/word-string {:max 6000}])
|
||||
;; ---- HELPERS
|
||||
|
||||
(defn email-domain-in-whitelist?
|
||||
"Returns true if email's domain is in the given whitelist or if
|
||||
given whitelist is an empty string."
|
||||
[domains email]
|
||||
(if (or (empty? domains)
|
||||
(nil? domains))
|
||||
true
|
||||
(let [[_ candidate] (-> (str/lower email)
|
||||
(str/split #"@" 2))]
|
||||
(contains? domains candidate))))
|
||||
|
||||
;; ---- COMMAND: login with password
|
||||
|
||||
@@ -98,22 +113,22 @@
|
||||
(rph/with-meta {::audit/props (audit/profile->props profile)
|
||||
::audit/profile-id (:id profile)}))))))
|
||||
|
||||
(def schema:login-with-password
|
||||
[:map {:title "login-with-password"}
|
||||
[:email ::sm/email]
|
||||
[:password schema:password]
|
||||
[:invitation-token {:optional true} schema:token]])
|
||||
(s/def ::login-with-password
|
||||
(s/keys :req-un [::email ::password]
|
||||
:opt-un [::invitation-token]))
|
||||
|
||||
(sv/defmethod ::login-with-password
|
||||
"Performs authentication using penpot password."
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::sm/params schema:login-with-password}
|
||||
::doc/added "1.15"}
|
||||
[cfg params]
|
||||
(login-with-password cfg params))
|
||||
|
||||
;; ---- COMMAND: Logout
|
||||
|
||||
(s/def ::logout
|
||||
(s/keys :opt [::rpc/profile-id]))
|
||||
|
||||
(sv/defmethod ::logout
|
||||
"Clears the authentication cookie and logout the current session."
|
||||
{::rpc/auth false
|
||||
@@ -138,15 +153,13 @@
|
||||
(update-password conn))
|
||||
nil)))
|
||||
|
||||
(def schema:recover-profile
|
||||
[:map {:title "recover-profile"}
|
||||
[:token schema:token]
|
||||
[:password schema:password]])
|
||||
(s/def ::token ::us/not-empty-string)
|
||||
(s/def ::recover-profile
|
||||
(s/keys :req-un [::token ::password]))
|
||||
|
||||
(sv/defmethod ::recover-profile
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::sm/params schema:recover-profile}
|
||||
::doc/added "1.15"}
|
||||
[cfg params]
|
||||
(recover-profile cfg params))
|
||||
|
||||
@@ -167,9 +180,10 @@
|
||||
:code :email-does-not-match-invitation
|
||||
:hint "email should match the invitation"))))
|
||||
|
||||
(when-not (auth/email-domain-in-whitelist? (:email params))
|
||||
(ex/raise :type :validation
|
||||
:code :email-domain-is-not-allowed))
|
||||
(when-let [domains (cf/get :registration-domain-whitelist)]
|
||||
(when-not (email-domain-in-whitelist? domains (:email params))
|
||||
(ex/raise :type :validation
|
||||
:code :email-domain-is-not-allowed)))
|
||||
|
||||
;; Don't allow proceed in preparing registration if the profile is
|
||||
;; already reported as spammer.
|
||||
@@ -227,16 +241,13 @@
|
||||
(with-meta {:token token}
|
||||
{::audit/profile-id uuid/zero})))
|
||||
|
||||
(def schema:prepare-register-profile
|
||||
[:map {:title "prepare-register-profile"}
|
||||
[:email ::sm/email]
|
||||
[:password schema:password]
|
||||
[:invitation-token {:optional true} schema:token]])
|
||||
(s/def ::prepare-register-profile
|
||||
(s/keys :req-un [::email ::password]
|
||||
:opt-un [::invitation-token]))
|
||||
|
||||
(sv/defmethod ::prepare-register-profile
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::sm/params schema:prepare-register-profile}
|
||||
::doc/added "1.15"}
|
||||
[cfg params]
|
||||
(prepare-register cfg params))
|
||||
|
||||
@@ -246,7 +257,7 @@
|
||||
"Create the profile entry on the database with limited set of input
|
||||
attrs (all the other attrs are filled with default values)."
|
||||
[conn {:keys [email] :as params}]
|
||||
(dm/assert! ::sm/email email)
|
||||
(us/assert! ::us/email email)
|
||||
(let [id (or (:id params) (uuid/next))
|
||||
props (-> (audit/extract-utm-params params)
|
||||
(merge (:props params))
|
||||
@@ -292,12 +303,9 @@
|
||||
|
||||
(defn create-profile-rels!
|
||||
[conn {:keys [id] :as profile}]
|
||||
(let [features (cfeat/get-enabled-features cf/flags)
|
||||
team (teams/create-team conn
|
||||
{:profile-id id
|
||||
:name "Default"
|
||||
:features features
|
||||
:is-default true})]
|
||||
(let [team (teams/create-team conn {:profile-id id
|
||||
:name "Default"
|
||||
:is-default true})]
|
||||
(-> (db/update! conn :profile
|
||||
{:default-team-id (:id team)
|
||||
:default-project-id (:default-project-id team)}
|
||||
@@ -327,9 +335,9 @@
|
||||
:extra-data ptoken})))
|
||||
|
||||
(defn register-profile
|
||||
[{:keys [::db/conn] :as cfg} {:keys [token fullname] :as params}]
|
||||
[{:keys [::db/conn] :as cfg} {:keys [token] :as params}]
|
||||
(let [claims (tokens/verify (::main/props cfg) {:token token :iss :prepared-register})
|
||||
params (assoc claims :fullname fullname)
|
||||
params (merge params claims)
|
||||
|
||||
is-active (or (:is-active params)
|
||||
(not (contains? cf/flags :email-verification)))
|
||||
@@ -396,16 +404,12 @@
|
||||
{::audit/replace-props (audit/profile->props profile)
|
||||
::audit/profile-id (:id profile)})))))
|
||||
|
||||
|
||||
(def schema:register-profile
|
||||
[:map {:title "register-profile"}
|
||||
[:token schema:token]
|
||||
[:fullname [::sm/word-string {:max 100}]]])
|
||||
(s/def ::register-profile
|
||||
(s/keys :req-un [::token ::fullname]))
|
||||
|
||||
(sv/defmethod ::register-profile
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::sm/params schema:register-profile}
|
||||
::doc/added "1.15"}
|
||||
[{:keys [::db/pool] :as cfg} params]
|
||||
(db/with-atomic [conn pool]
|
||||
(-> (assoc cfg ::db/conn conn)
|
||||
@@ -457,15 +461,12 @@
|
||||
(create-recovery-token)
|
||||
(send-email-notification conn))))))
|
||||
|
||||
|
||||
(def schema:request-profile-recovery
|
||||
[:map {:title "request-profile-recovery"}
|
||||
[:email ::sm/email]])
|
||||
(s/def ::request-profile-recovery
|
||||
(s/keys :req-un [::email]))
|
||||
|
||||
(sv/defmethod ::request-profile-recovery
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::sm/params schema:request-profile-recovery}
|
||||
::doc/added "1.15"}
|
||||
[cfg params]
|
||||
(request-profile-recovery cfg params))
|
||||
|
||||
|
||||
@@ -8,47 +8,38 @@
|
||||
(:refer-clojure :exclude [assert])
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.defaults :as cfd]
|
||||
[app.common.files.migrations :as fmg]
|
||||
[app.common.files.validate :as fval]
|
||||
[app.common.files.features :as ffeat]
|
||||
[app.common.fressian :as fres]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.components-v2 :as feat.compv2]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.http.sse :as sse]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.media :as media]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.projects :as projects]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.tasks.file-gc]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.objects-map :as omap]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.set :as set]
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.walk :as walk]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.io :as io]
|
||||
[promesa.core :as p]
|
||||
[promesa.util :as pu]
|
||||
[ring.response :as rres]
|
||||
[yetti.adapter :as yt])
|
||||
[yetti.adapter :as yt]
|
||||
[yetti.response :as yrs])
|
||||
(:import
|
||||
com.github.luben.zstd.ZstdInputStream
|
||||
com.github.luben.zstd.ZstdOutputStream
|
||||
@@ -303,46 +294,30 @@
|
||||
[output & {:keys [level] :or {level 0}}]
|
||||
(ZstdOutputStream. ^OutputStream output (int level)))
|
||||
|
||||
(defn- get-files
|
||||
[cfg ids]
|
||||
(db/run! cfg (fn [{:keys [::db/conn]}]
|
||||
(let [sql (str "SELECT id FROM file "
|
||||
" WHERE id = ANY(?) ")
|
||||
ids (db/create-array conn "uuid" ids)]
|
||||
(->> (db/exec! conn [sql ids])
|
||||
(into [] (map :id))
|
||||
(not-empty))))))
|
||||
(defn- retrieve-file
|
||||
[pool file-id]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
|
||||
(some-> (db/get* conn :file {:id file-id})
|
||||
(files/decode-row)
|
||||
(files/process-pointers deref)))))
|
||||
|
||||
(defn- get-file
|
||||
[cfg file-id]
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
|
||||
(some-> (db/get* conn :file {:id file-id} {::db/remove-deleted? false})
|
||||
(files/decode-row)
|
||||
(update :data feat.fdata/process-pointers deref))))))
|
||||
(def ^:private sql:file-media-objects
|
||||
"SELECT * FROM file_media_object WHERE id = ANY(?)")
|
||||
|
||||
(defn- get-file-media
|
||||
[{:keys [::db/pool]} {:keys [data id] :as file}]
|
||||
(pu/with-open [conn (db/open pool)]
|
||||
(defn- retrieve-file-media
|
||||
[pool {:keys [data id] :as file}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(let [ids (app.tasks.file-gc/collect-used-media data)
|
||||
ids (db/create-array conn "uuid" ids)
|
||||
sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")]
|
||||
ids (db/create-array conn "uuid" ids)]
|
||||
|
||||
;; We assoc the file-id again to the file-media-object row
|
||||
;; because there are cases that used objects refer to other
|
||||
;; files and we need to ensure in the exportation process that
|
||||
;; all ids matches
|
||||
(->> (db/exec! conn [sql ids])
|
||||
(->> (db/exec! conn [sql:file-media-objects ids])
|
||||
(mapv #(assoc % :file-id id))))))
|
||||
|
||||
(defn- get-file-thumbnails
|
||||
"Return all file thumbnails for a given file."
|
||||
[{:keys [::db/pool]} id]
|
||||
(pu/with-open [conn (db/open pool)]
|
||||
(let [sql "SELECT * FROM file_tagged_object_thumbnail WHERE file_id = ?"]
|
||||
(->> (db/exec! conn [sql id])
|
||||
(mapv #(dissoc % :file-id))))))
|
||||
|
||||
(def ^:private storage-object-id-xf
|
||||
(comp
|
||||
(mapcat (juxt :media-id :thumbnail-id))
|
||||
@@ -350,34 +325,36 @@
|
||||
|
||||
(def ^:private sql:file-libraries
|
||||
"WITH RECURSIVE libs AS (
|
||||
SELECT fl.id
|
||||
SELECT fl.id, fl.deleted_at
|
||||
FROM file AS fl
|
||||
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
|
||||
WHERE flr.file_id = ANY(?)
|
||||
UNION
|
||||
SELECT fl.id
|
||||
SELECT fl.id, fl.deleted_at
|
||||
FROM file AS fl
|
||||
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
|
||||
JOIN libs AS l ON (flr.file_id = l.id)
|
||||
)
|
||||
SELECT DISTINCT l.id
|
||||
FROM libs AS l")
|
||||
FROM libs AS l
|
||||
WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
|
||||
|
||||
(defn- get-libraries
|
||||
[{:keys [::db/pool]} ids]
|
||||
(pu/with-open [conn (db/open pool)]
|
||||
(defn- retrieve-libraries
|
||||
[pool ids]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(map :id (db/exec! pool [sql:file-libraries ids])))))
|
||||
|
||||
(defn- get-library-relations
|
||||
[cfg ids]
|
||||
(db/run! cfg (fn [{:keys [::db/conn]}]
|
||||
(let [ids (db/create-array conn "uuid" ids)
|
||||
sql (str "SELECT flr.* FROM file_library_rel AS flr "
|
||||
" WHERE flr.file_id = ANY(?)")]
|
||||
(db/exec! conn [sql ids])))))
|
||||
(def ^:private sql:file-library-rels
|
||||
"SELECT * FROM file_library_rel
|
||||
WHERE file_id = ANY(?)")
|
||||
|
||||
(defn- create-or-update-file!
|
||||
(defn- retrieve-library-relations
|
||||
[pool ids]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(db/exec! conn [sql:file-library-rels (db/create-array conn "uuid" ids)])))
|
||||
|
||||
(defn- create-or-update-file
|
||||
[conn params]
|
||||
(let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
@@ -395,13 +372,13 @@
|
||||
|
||||
;; --- GENERAL PURPOSE DYNAMIC VARS
|
||||
|
||||
(def ^:dynamic *state* nil)
|
||||
(def ^:dynamic *options* nil)
|
||||
(def ^:dynamic *state*)
|
||||
(def ^:dynamic *options*)
|
||||
|
||||
;; --- EXPORT WRITER
|
||||
|
||||
(defn- embed-file-assets
|
||||
[data cfg file-id]
|
||||
[data conn file-id]
|
||||
(letfn [(walk-map-form [form state]
|
||||
(cond
|
||||
(uuid? (:fill-color-ref-file form))
|
||||
@@ -431,7 +408,7 @@
|
||||
;; NOTE: there is a possibility that shape refers to an
|
||||
;; non-existant file because the file was removed. In this
|
||||
;; case we just ignore the asset.
|
||||
(if-let [lib (get-file cfg lib-id)]
|
||||
(if-let [lib (retrieve-file conn lib-id)]
|
||||
(reduce (partial process-asset lib) data items)
|
||||
data))
|
||||
|
||||
@@ -484,104 +461,78 @@
|
||||
(defmethod write-export :default
|
||||
[{:keys [::output] :as options}]
|
||||
(write-header! output :v1)
|
||||
(pu/with-open [output (zstd-output-stream output :level 12)
|
||||
output (io/data-output-stream output)]
|
||||
(binding [*state* (volatile! {})]
|
||||
(run! (fn [section]
|
||||
(l/dbg :hint "write section" :section section ::l/sync? true)
|
||||
(write-label! output section)
|
||||
(let [options (-> options
|
||||
(assoc ::output output)
|
||||
(assoc ::section section))]
|
||||
(binding [*options* options]
|
||||
(write-section options))))
|
||||
(with-open [output (zstd-output-stream output :level 12)]
|
||||
(with-open [output (io/data-output-stream output)]
|
||||
(binding [*state* (volatile! {})]
|
||||
(run! (fn [section]
|
||||
(l/debug :hint "write section" :section section ::l/sync? true)
|
||||
(write-label! output section)
|
||||
(let [options (-> options
|
||||
(assoc ::output output)
|
||||
(assoc ::section section))]
|
||||
(binding [*options* options]
|
||||
(write-section options))))
|
||||
|
||||
[:v1/metadata :v1/files :v1/rels :v1/sobjects]))))
|
||||
[:v1/metadata :v1/files :v1/rels :v1/sobjects])))))
|
||||
|
||||
(defmethod write-section :v1/metadata
|
||||
[{:keys [::output ::file-ids ::include-libraries?] :as cfg}]
|
||||
(if-let [fids (get-files cfg file-ids)]
|
||||
(let [lids (when include-libraries?
|
||||
(get-libraries cfg file-ids))
|
||||
ids (into fids lids)]
|
||||
(write-obj! output {:version cf/version :files ids})
|
||||
(vswap! *state* assoc :files ids))
|
||||
(ex/raise :type :not-found
|
||||
:code :files-not-found
|
||||
:hint "unable to retrieve files for export")))
|
||||
[{:keys [::db/pool ::output ::file-ids ::include-libraries?]}]
|
||||
(let [libs (when include-libraries?
|
||||
(retrieve-libraries pool file-ids))
|
||||
files (into file-ids libs)]
|
||||
(write-obj! output {:version cf/version :files files})
|
||||
(vswap! *state* assoc :files files)))
|
||||
|
||||
(defmethod write-section :v1/files
|
||||
[{:keys [::output ::embed-assets? ::include-libraries?] :as cfg}]
|
||||
[{:keys [::db/pool ::output ::embed-assets?]}]
|
||||
|
||||
;; Initialize SIDS with empty vector
|
||||
(vswap! *state* assoc :sids [])
|
||||
|
||||
(doseq [file-id (-> *state* deref :files)]
|
||||
(let [detach? (and (not embed-assets?) (not include-libraries?))
|
||||
thumbnails (get-file-thumbnails cfg file-id)
|
||||
file (cond-> (get-file cfg file-id)
|
||||
detach?
|
||||
(-> (ctf/detach-external-references file-id)
|
||||
(dissoc :libraries))
|
||||
(let [file (cond-> (retrieve-file pool file-id)
|
||||
embed-assets?
|
||||
(update :data embed-file-assets pool file-id))
|
||||
|
||||
embed-assets?
|
||||
(update :data embed-file-assets cfg file-id)
|
||||
media (retrieve-file-media pool file)]
|
||||
|
||||
:always
|
||||
(assoc :thumbnails thumbnails))
|
||||
|
||||
media (get-file-media cfg file)]
|
||||
|
||||
(l/dbg :hint "write penpot file"
|
||||
:id (str file-id)
|
||||
:name (:name file)
|
||||
:thumbnails (count thumbnails)
|
||||
:features (:features file)
|
||||
:media (count media)
|
||||
::l/sync? true)
|
||||
|
||||
(doseq [item media]
|
||||
(l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true))
|
||||
|
||||
(doseq [item thumbnails]
|
||||
(l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true))
|
||||
(l/debug :hint "write penpot file"
|
||||
:id file-id
|
||||
:media (count media)
|
||||
::l/sync? true)
|
||||
|
||||
(doto output
|
||||
(write-obj! file)
|
||||
(write-obj! media))
|
||||
|
||||
(vswap! *state* update :sids into storage-object-id-xf media)
|
||||
(vswap! *state* update :sids into storage-object-id-xf thumbnails))))
|
||||
(vswap! *state* update :sids into storage-object-id-xf media))))
|
||||
|
||||
(defmethod write-section :v1/rels
|
||||
[{:keys [::output ::include-libraries?] :as cfg}]
|
||||
(let [ids (-> *state* deref :files)
|
||||
rels (when include-libraries?
|
||||
(get-library-relations cfg ids))]
|
||||
(l/dbg :hint "found rels" :total (count rels) ::l/sync? true)
|
||||
[{:keys [::db/pool ::output ::include-libraries?]}]
|
||||
(let [rels (when include-libraries?
|
||||
(retrieve-library-relations pool (-> *state* deref :files)))]
|
||||
(l/debug :hint "found rels" :total (count rels) ::l/sync? true)
|
||||
(write-obj! output rels)))
|
||||
|
||||
(defmethod write-section :v1/sobjects
|
||||
[{:keys [::sto/storage ::output]}]
|
||||
(let [sids (-> *state* deref :sids)
|
||||
storage (media/configure-assets-storage storage)]
|
||||
|
||||
(l/dbg :hint "found sobjects"
|
||||
:items (count sids)
|
||||
::l/sync? true)
|
||||
(l/debug :hint "found sobjects"
|
||||
:items (count sids)
|
||||
::l/sync? true)
|
||||
|
||||
;; Write all collected storage objects
|
||||
(write-obj! output sids)
|
||||
|
||||
(doseq [id sids]
|
||||
(let [{:keys [size] :as obj} (sto/get-object storage id)]
|
||||
(l/dbg :hint "write sobject" :id (str id) ::l/sync? true)
|
||||
|
||||
(l/debug :hint "write sobject" :id id ::l/sync? true)
|
||||
(doto output
|
||||
(write-uuid! id)
|
||||
(write-obj! (meta obj)))
|
||||
|
||||
(pu/with-open [stream (sto/get-object-data storage obj)]
|
||||
(with-open [^InputStream stream (sto/get-object-data storage obj)]
|
||||
(let [written (write-stream! output stream size)]
|
||||
(when (not= written size)
|
||||
(ex/raise :type :validation
|
||||
@@ -598,16 +549,15 @@
|
||||
(defmulti read-import ::version)
|
||||
(defmulti read-section ::section)
|
||||
|
||||
(s/def ::profile-id ::us/uuid)
|
||||
(s/def ::project-id ::us/uuid)
|
||||
(s/def ::input io/input-stream?)
|
||||
(s/def ::overwrite? (s/nilable ::us/boolean))
|
||||
(s/def ::migrate? (s/nilable ::us/boolean))
|
||||
(s/def ::ignore-index-errors? (s/nilable ::us/boolean))
|
||||
|
||||
;; FIXME: replace with schema
|
||||
(s/def ::read-import-options
|
||||
(s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input]
|
||||
:opt [::overwrite? ::ignore-index-errors?]))
|
||||
(s/keys :req [::db/pool ::sto/storage ::project-id ::input]
|
||||
:opt [::overwrite? ::migrate? ::ignore-index-errors?]))
|
||||
|
||||
(defn read-import!
|
||||
"Do the importation of the specified resource in penpot custom binary
|
||||
@@ -617,6 +567,9 @@
|
||||
`::overwrite?`: if true, instead of creating new files and remapping id references,
|
||||
it reuses all ids and updates existing objects; defaults to `false`.
|
||||
|
||||
`::migrate?`: if true, applies the migration before persisting the
|
||||
file data; defaults to `false`.
|
||||
|
||||
`::ignore-index-errors?`: if true, do not fail on index lookup errors, can
|
||||
happen with broken files; defaults to: `false`.
|
||||
"
|
||||
@@ -626,233 +579,123 @@
|
||||
(let [version (read-header! input)]
|
||||
(read-import (assoc options ::version version ::timestamp timestamp))))
|
||||
|
||||
(defn- read-import-v1
|
||||
[{:keys [::db/conn ::project-id ::profile-id ::input] :as options}]
|
||||
(db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"])
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
|
||||
|
||||
(pu/with-open [input (zstd-input-stream input)
|
||||
input (io/data-input-stream input)]
|
||||
(binding [*state* (volatile! {:media [] :index {}})]
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:project-id project-id)
|
||||
|
||||
validate? (contains? cf/flags :file-validation)
|
||||
features (cfeat/get-team-enabled-features cf/flags team)]
|
||||
|
||||
(sse/tap {:type :import-progress
|
||||
:section :read-import})
|
||||
|
||||
;; Process all sections
|
||||
(run! (fn [section]
|
||||
(l/dbg :hint "reading section" :section section ::l/sync? true)
|
||||
(assert-read-label! input section)
|
||||
(let [options (-> options
|
||||
(assoc ::enabled-features features)
|
||||
(assoc ::section section)
|
||||
(assoc ::input input))]
|
||||
(binding [*options* options]
|
||||
(sse/tap {:type :import-progress
|
||||
:section section})
|
||||
(read-section options))))
|
||||
[:v1/metadata :v1/files :v1/rels :v1/sobjects])
|
||||
|
||||
;; Run all pending migrations
|
||||
(doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
|
||||
(case feature
|
||||
"components/v2"
|
||||
(feat.compv2/migrate-file! options file-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? true)
|
||||
|
||||
"fdata/shape-data-type"
|
||||
nil
|
||||
|
||||
(ex/raise :type :internal
|
||||
:code :no-migration-defined
|
||||
:hint (str/ffmt "no migation for feature '%' on file importation" feature)
|
||||
:feature feature)))
|
||||
|
||||
;; Knowing that the ids of the created files are in index,
|
||||
;; just lookup them and return it as a set
|
||||
(let [files (-> *state* deref :files)]
|
||||
(into #{} (keep #(get-in @*state* [:index %])) files))))))
|
||||
|
||||
(defmethod read-import :v1
|
||||
[options]
|
||||
(db/tx-run! options read-import-v1))
|
||||
[{:keys [::db/pool ::input] :as options}]
|
||||
(with-open [input (zstd-input-stream input)]
|
||||
(with-open [input (io/data-input-stream input)]
|
||||
(db/with-atomic [conn pool]
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
|
||||
(binding [*state* (volatile! {:media [] :index {}})]
|
||||
(run! (fn [section]
|
||||
(l/debug :hint "reading section" :section section ::l/sync? true)
|
||||
(assert-read-label! input section)
|
||||
(let [options (-> options
|
||||
(assoc ::section section)
|
||||
(assoc ::input input)
|
||||
(assoc :conn conn))]
|
||||
(binding [*options* options]
|
||||
(read-section options))))
|
||||
[:v1/metadata :v1/files :v1/rels :v1/sobjects])
|
||||
|
||||
;; Knowing that the ids of the created files are in
|
||||
;; index, just lookup them and return it as a set
|
||||
(let [files (-> *state* deref :files)]
|
||||
(into #{} (keep #(get-in @*state* [:index %])) files)))))))
|
||||
|
||||
(defmethod read-section :v1/metadata
|
||||
[{:keys [::input]}]
|
||||
(let [{:keys [version files]} (read-obj! input)]
|
||||
(l/dbg :hint "metadata readed"
|
||||
:version (:full version)
|
||||
:files (mapv str files)
|
||||
::l/sync? true)
|
||||
(l/debug :hint "metadata readed" :version (:full version) :files files ::l/sync? true)
|
||||
(vswap! *state* update :index update-index files)
|
||||
(vswap! *state* assoc :version version :files files)))
|
||||
|
||||
(defn- get-remaped-thumbnails
|
||||
[thumbnails file-id]
|
||||
(mapv (fn [thumbnail]
|
||||
(-> thumbnail
|
||||
(assoc :file-id file-id)
|
||||
(update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/")))))
|
||||
thumbnails))
|
||||
|
||||
(defn- process-file
|
||||
[{:keys [id] :as file}]
|
||||
(-> file
|
||||
(update :data (fn [fdata]
|
||||
(-> fdata
|
||||
(assoc :id id)
|
||||
(dissoc :recent-colors)
|
||||
(cond-> (> (:version fdata) cfd/version)
|
||||
(assoc :version cfd/version))
|
||||
;; FIXME: We're temporarily activating all
|
||||
;; migrations because a problem in the
|
||||
;; environments messed up with the version
|
||||
;; numbers When this problem is fixed delete
|
||||
;; the following line
|
||||
(cond-> (> (:version fdata) 22)
|
||||
(assoc :version 22)))))
|
||||
(fmg/migrate-file)
|
||||
(update :data (fn [fdata]
|
||||
(-> fdata
|
||||
(update :pages-index relink-shapes)
|
||||
(update :components relink-shapes)
|
||||
(update :media relink-media)
|
||||
(d/without-nils))))))
|
||||
|
||||
(defn- postprocess-file
|
||||
[data]
|
||||
(let [omap-wrap ffeat/*wrap-with-objects-map-fn*
|
||||
pmap-wrap ffeat/*wrap-with-pointer-map-fn*]
|
||||
(-> data
|
||||
(update :pages-index update-vals #(update % :objects omap-wrap))
|
||||
(update :pages-index update-vals pmap-wrap)
|
||||
(update :components update-vals #(d/update-when % :objects omap-wrap))
|
||||
(update :components pmap-wrap))))
|
||||
|
||||
(defmethod read-section :v1/files
|
||||
[{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite?] :as system}]
|
||||
|
||||
[{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
|
||||
(doseq [expected-file-id (-> *state* deref :files)]
|
||||
(let [file (read-obj! input)
|
||||
media (read-obj! input)
|
||||
|
||||
file-id (:id file)
|
||||
file-id' (lookup-index file-id)
|
||||
|
||||
thumbnails (:thumbnails file)]
|
||||
(let [file (read-obj! input)
|
||||
media' (read-obj! input)
|
||||
file-id (:id file)
|
||||
features (files/get-default-features)]
|
||||
|
||||
(when (not= file-id expected-file-id)
|
||||
(ex/raise :type :validation
|
||||
:code :inconsistent-penpot-file
|
||||
:found-id file-id
|
||||
:expected-id expected-file-id
|
||||
:hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
|
||||
|
||||
(l/dbg :hint "processing file"
|
||||
:id (str file-id)
|
||||
:features (:features file)
|
||||
:version (-> file :data :version)
|
||||
:media (count media)
|
||||
:thumbnails (count thumbnails)
|
||||
::l/sync? true)
|
||||
;; Update index using with media
|
||||
(l/debug :hint "update index with media" ::l/sync? true)
|
||||
(vswap! *state* update :index update-index (map :id media'))
|
||||
|
||||
(when (seq thumbnails)
|
||||
(let [thumbnails (get-remaped-thumbnails thumbnails file-id')]
|
||||
(l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true)
|
||||
(vswap! *state* update :thumbnails (fnil into []) thumbnails)))
|
||||
;; Store file media for later insertion
|
||||
(l/debug :hint "update media references" ::l/sync? true)
|
||||
(vswap! *state* update :media into (map #(update % :id lookup-index)) media')
|
||||
|
||||
(when (seq media)
|
||||
;; Update index with media
|
||||
(l/dbg :hint "update index with media" :total (count media) ::l/sync? true)
|
||||
(vswap! *state* update :index update-index (map :id media))
|
||||
(l/debug :hint "processing file" :file-id file-id ::features features ::l/sync? true)
|
||||
|
||||
;; Store file media for later insertion
|
||||
(l/dbg :hint "update media references" ::l/sync? true)
|
||||
(vswap! *state* update :media into (map #(update % :id lookup-index)) media))
|
||||
(binding [ffeat/*current* features
|
||||
ffeat/*wrap-with-objects-map-fn* (if (features "storage/objects-map") omap/wrap identity)
|
||||
ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)
|
||||
pmap/*tracked* (atom {})]
|
||||
|
||||
(let [file (-> file
|
||||
(assoc :id file-id')
|
||||
(process-file))
|
||||
(let [file-id' (lookup-index file-id)
|
||||
data (-> (:data file)
|
||||
(assoc :id file-id')
|
||||
(cond-> migrate? (pmg/migrate-data))
|
||||
(update :pages-index relink-shapes)
|
||||
(update :components relink-shapes)
|
||||
(update :media relink-media)
|
||||
(postprocess-file))
|
||||
|
||||
;; All features that are enabled and requires explicit migration are
|
||||
;; added to the state for a posterior migration step.
|
||||
_ (doseq [feature (-> enabled-features
|
||||
(set/difference cfeat/no-migration-features)
|
||||
(set/difference (:features file)))]
|
||||
(vswap! *state* update :pending-to-migrate (fnil conj []) [feature file-id']))
|
||||
params {:id file-id'
|
||||
:project-id project-id
|
||||
:features (db/create-array conn "text" features)
|
||||
:name (:name file)
|
||||
:revn (:revn file)
|
||||
:is-shared (:is-shared file)
|
||||
:data (blob/encode data)
|
||||
:created-at timestamp
|
||||
:modified-at timestamp}]
|
||||
|
||||
file (-> file
|
||||
(assoc :project-id project-id)
|
||||
(assoc :created-at timestamp)
|
||||
(assoc :modified-at timestamp)
|
||||
(dissoc :thumbnails)
|
||||
(update :features
|
||||
(fn [features]
|
||||
(let [features (cfeat/check-supported-features! features)]
|
||||
(-> enabled-features
|
||||
(set/difference cfeat/frontend-only-features)
|
||||
(set/union features))))))
|
||||
|
||||
_ (when (contains? cf/flags :file-schema-validation)
|
||||
(fval/validate-file-schema! file))
|
||||
|
||||
_ (when (contains? cf/flags :soft-file-schema-validation)
|
||||
(let [result (ex/try! (fval/validate-file-schema! file))]
|
||||
(when (ex/exception? result)
|
||||
(l/error :hint "file schema validation error" :cause result))))
|
||||
|
||||
file (if (contains? (:features file) "fdata/objects-map")
|
||||
(feat.fdata/enable-objects-map file)
|
||||
file)
|
||||
|
||||
file (if (contains? (:features file) "fdata/pointer-map")
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)]
|
||||
(let [file (feat.fdata/enable-pointer-map file)]
|
||||
(feat.fdata/persist-pointers! system file-id')
|
||||
file))
|
||||
file)
|
||||
|
||||
file (-> file
|
||||
(update :features #(db/create-array conn "text" %))
|
||||
(update :data blob/encode))]
|
||||
|
||||
(l/dbg :hint "create file" :id (str file-id') ::l/sync? true)
|
||||
(l/debug :hint "create file" :id file-id' ::l/sync? true)
|
||||
|
||||
(if overwrite?
|
||||
(create-or-update-file! conn file)
|
||||
(db/insert! conn :file file))
|
||||
(create-or-update-file conn params)
|
||||
(db/insert! conn :file params))
|
||||
|
||||
(files/persist-pointers! conn file-id')
|
||||
|
||||
(when overwrite?
|
||||
(db/delete! conn :file-thumbnail {:file-id file-id'}))
|
||||
|
||||
file-id'))))
|
||||
(db/delete! conn :file-thumbnail {:file-id file-id'})))))))
|
||||
|
||||
(defmethod read-section :v1/rels
|
||||
[{:keys [::db/conn ::input ::timestamp]}]
|
||||
(let [rels (read-obj! input)
|
||||
ids (into #{} (-> *state* deref :files))]
|
||||
[{:keys [conn ::input ::timestamp]}]
|
||||
(let [rels (read-obj! input)]
|
||||
;; Insert all file relations
|
||||
(doseq [{:keys [library-file-id] :as rel} rels]
|
||||
(doseq [rel rels]
|
||||
(let [rel (-> rel
|
||||
(assoc :synced-at timestamp)
|
||||
(update :file-id lookup-index)
|
||||
(update :library-file-id lookup-index))]
|
||||
|
||||
(if (contains? ids library-file-id)
|
||||
(do
|
||||
(l/dbg :hint "create file library link"
|
||||
:file-id (:file-id rel)
|
||||
:lib-id (:library-file-id rel)
|
||||
::l/sync? true)
|
||||
(db/insert! conn :file-library-rel rel))
|
||||
|
||||
(l/warn :hint "ignoring file library link"
|
||||
:file-id (:file-id rel)
|
||||
:lib-id (:library-file-id rel)
|
||||
::l/sync? true))))))
|
||||
(l/debug :hint "create file library link"
|
||||
:file-id (:file-id rel)
|
||||
:lib-id (:library-file-id rel)
|
||||
::l/sync? true)
|
||||
(db/insert! conn :file-library-rel rel)))))
|
||||
|
||||
(defmethod read-section :v1/sobjects
|
||||
[{:keys [::sto/storage ::db/conn ::input ::overwrite? ::timestamp]}]
|
||||
[{:keys [::sto/storage conn ::input ::overwrite?]}]
|
||||
(let [storage (media/configure-assets-storage storage)
|
||||
ids (read-obj! input)
|
||||
thumb? (into #{} (map :media-id) (:thumbnails @*state*))]
|
||||
ids (read-obj! input)]
|
||||
|
||||
(doseq [expected-storage-id ids]
|
||||
(let [id (read-uuid! input)
|
||||
@@ -863,62 +706,43 @@
|
||||
:code :inconsistent-penpot-file
|
||||
:hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
|
||||
|
||||
(l/dbg :hint "readed storage object" :id (str id) ::l/sync? true)
|
||||
(l/debug :hint "readed storage object" :id id ::l/sync? true)
|
||||
|
||||
(let [[size resource] (read-stream! input)
|
||||
hash (sto/calculate-hash resource)
|
||||
content (-> (sto/content resource size)
|
||||
(sto/wrap-with-hash hash))
|
||||
|
||||
params (-> mdata
|
||||
(assoc ::sto/content content)
|
||||
(assoc ::sto/deduplicate? true)
|
||||
(assoc ::sto/touched-at timestamp))
|
||||
|
||||
params (if (thumb? id)
|
||||
(assoc params :bucket "file-object-thumbnail")
|
||||
(assoc params :bucket "file-media-object"))
|
||||
(assoc ::sto/content content)
|
||||
(assoc ::sto/touched-at (dt/now))
|
||||
(assoc :bucket "file-media-object"))
|
||||
|
||||
sobject (sto/put-object! storage params)]
|
||||
|
||||
(l/dbg :hint "persisted storage object"
|
||||
:old-id (str id)
|
||||
:new-id (str (:id sobject))
|
||||
:is-thumbnail (boolean (thumb? id))
|
||||
::l/sync? true)
|
||||
|
||||
(l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/sync? true)
|
||||
(vswap! *state* update :index assoc id (:id sobject)))))
|
||||
|
||||
(doseq [item (:media @*state*)]
|
||||
(l/dbg :hint "inserting file media object"
|
||||
:id (str (:id item))
|
||||
:file-id (str (:file-id item))
|
||||
::l/sync? true)
|
||||
(l/debug :hint "inserting file media object"
|
||||
:id (:id item)
|
||||
:file-id (:file-id item)
|
||||
::l/sync? true)
|
||||
|
||||
(let [file-id (lookup-index (:file-id item))]
|
||||
(if (= file-id (:file-id item))
|
||||
(l/warn :hint "ignoring file media object" :file-id (str file-id) ::l/sync? true)
|
||||
(l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/sync? true)
|
||||
(db/insert! conn :file-media-object
|
||||
(-> item
|
||||
(assoc :file-id file-id)
|
||||
(d/update-when :media-id lookup-index)
|
||||
(d/update-when :thumbnail-id lookup-index))
|
||||
{::db/on-conflict-do-nothing? overwrite?}))))
|
||||
|
||||
(doseq [item (:thumbnails @*state*)]
|
||||
(let [item (update item :media-id lookup-index)]
|
||||
(l/dbg :hint "inserting file object thumbnail"
|
||||
:file-id (str (:file-id item))
|
||||
:media-id (str (:media-id item))
|
||||
:object-id (:object-id item)
|
||||
::l/sync? true)
|
||||
(db/insert! conn :file-tagged-object-thumbnail item
|
||||
{::db/on-conflict-do-nothing? overwrite?})))))
|
||||
{:on-conflict-do-nothing overwrite?}))))))
|
||||
|
||||
(defn- lookup-index
|
||||
[id]
|
||||
(let [val (get-in @*state* [:index id])]
|
||||
(l/trc :fn "lookup-index" :id id :val val ::l/sync? true)
|
||||
(l/trace :fn "lookup-index" :id id :val val ::l/sync? true)
|
||||
(when (and (not (::ignore-index-errors? *options*)) (not val))
|
||||
(ex/raise :type :validation
|
||||
:code :incomplete-index
|
||||
@@ -931,7 +755,7 @@
|
||||
index index]
|
||||
(if-let [id (first items)]
|
||||
(let [new-id (if (::overwrite? *options*) id (uuid/next))]
|
||||
(l/trc :fn "update-index" :id id :new-id new-id ::l/sync? true)
|
||||
(l/trace :fn "update-index" :id id :new-id new-id ::l/sync? true)
|
||||
(recur (rest items)
|
||||
(assoc index id new-id)))
|
||||
index)))
|
||||
@@ -949,7 +773,8 @@
|
||||
(update-in [:metadata :id] lookup-index)
|
||||
|
||||
;; Relink paths with fill image
|
||||
(map? (:fill-image form))
|
||||
(and (map? (:fill-image form))
|
||||
(= :path (:type form)))
|
||||
(update-in [:fill-image :id] lookup-index)
|
||||
|
||||
;; This covers old shapes and the new :fills.
|
||||
@@ -1008,8 +833,8 @@
|
||||
ab (volatile! false)
|
||||
cs (volatile! nil)]
|
||||
(try
|
||||
(l/info :hint "start exportation" :export-id (str id))
|
||||
(pu/with-open [output (io/output-stream output)]
|
||||
(l/info :hint "start exportation" :export-id id)
|
||||
(dm/with-open [output (io/output-stream output)]
|
||||
(binding [*position* (atom 0)]
|
||||
(write-export! (assoc cfg ::output output))))
|
||||
|
||||
@@ -1024,7 +849,7 @@
|
||||
(throw cause))
|
||||
|
||||
(finally
|
||||
(l/info :hint "exportation finished" :export-id (str id)
|
||||
(l/info :hint "exportation finished" :export-id id
|
||||
:elapsed (str (inst-ms (tp)) "ms")
|
||||
:aborted @ab
|
||||
:cause @cs)))))
|
||||
@@ -1032,7 +857,7 @@
|
||||
(defn export-to-tmpfile!
|
||||
[cfg]
|
||||
(let [path (tmp/tempfile :prefix "penpot.export.")]
|
||||
(pu/with-open [output (io/output-stream path)]
|
||||
(dm/with-open [output (io/output-stream path)]
|
||||
(export! cfg output)
|
||||
path)))
|
||||
|
||||
@@ -1041,10 +866,10 @@
|
||||
(let [id (uuid/next)
|
||||
tp (dt/tpoint)
|
||||
cs (volatile! nil)]
|
||||
(l/info :hint "import: started" :id (str id))
|
||||
(l/info :hint "import: started" :import-id id)
|
||||
(try
|
||||
(binding [*position* (atom 0)]
|
||||
(pu/with-open [input (io/input-stream input)]
|
||||
(dm/with-open [input (io/input-stream input)]
|
||||
(read-import! (assoc cfg ::input input))))
|
||||
|
||||
(catch Throwable cause
|
||||
@@ -1053,78 +878,56 @@
|
||||
|
||||
(finally
|
||||
(l/info :hint "import: terminated"
|
||||
:id (str id)
|
||||
:import-id id
|
||||
:elapsed (dt/format-duration (tp))
|
||||
:error? (some? @cs))))))
|
||||
|
||||
:error? (some? @cs)
|
||||
:cause @cs
|
||||
)))))
|
||||
|
||||
;; --- Command: export-binfile
|
||||
|
||||
(def ^:private
|
||||
schema:export-binfile
|
||||
(sm/define
|
||||
[:map {:title "export-binfile"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:include-libraries? :boolean]
|
||||
[:embed-assets? :boolean]]))
|
||||
(s/def ::file-id ::us/uuid)
|
||||
(s/def ::include-libraries? ::us/boolean)
|
||||
(s/def ::embed-assets? ::us/boolean)
|
||||
|
||||
(s/def ::export-binfile
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::include-libraries? ::embed-assets?]))
|
||||
|
||||
(sv/defmethod ::export-binfile
|
||||
"Export a penpot file in a binary format."
|
||||
{::doc/added "1.15"
|
||||
::webhooks/event? true
|
||||
::sm/result schema:export-binfile}
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries? embed-assets?] :as params}]
|
||||
(files/check-read-permissions! pool profile-id file-id)
|
||||
(fn [_]
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "application/octet-stream"}
|
||||
::rres/body (reify rres/StreamableResponseBody
|
||||
(-write-body-to-stream [_ _ output-stream]
|
||||
(-> cfg
|
||||
(assoc ::file-ids [file-id])
|
||||
(assoc ::embed-assets? embed-assets?)
|
||||
(assoc ::include-libraries? include-libraries?)
|
||||
(export! output-stream))))}))
|
||||
(let [body (reify yrs/StreamableResponseBody
|
||||
(-write-body-to-stream [_ _ output-stream]
|
||||
(-> cfg
|
||||
(assoc ::file-ids [file-id])
|
||||
(assoc ::embed-assets? embed-assets?)
|
||||
(assoc ::include-libraries? include-libraries?)
|
||||
(export! output-stream))))]
|
||||
|
||||
(fn [_]
|
||||
{::yrs/status 200
|
||||
::yrs/body body
|
||||
::yrs/headers {"content-type" "application/octet-stream"}})))
|
||||
|
||||
;; --- Command: import-binfile
|
||||
|
||||
(def ^:private
|
||||
schema:import-binfile
|
||||
(sm/define
|
||||
[:map {:title "import-binfile"}
|
||||
[:project-id ::sm/uuid]
|
||||
[:file ::media/upload]]))
|
||||
|
||||
(declare ^:private import-binfile)
|
||||
(s/def ::file ::media/upload)
|
||||
(s/def ::import-binfile
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::project-id ::file]))
|
||||
|
||||
(sv/defmethod ::import-binfile
|
||||
"Import a penpot file in a binary format."
|
||||
{::doc/added "1.15"
|
||||
::webhooks/event? true
|
||||
::sse/stream? true
|
||||
::sm/params schema:import-binfile}
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id file] :as params}]
|
||||
(projects/check-read-permissions! pool profile-id project-id)
|
||||
(let [params (-> cfg
|
||||
(assoc ::input (:path file))
|
||||
(assoc ::project-id project-id)
|
||||
(assoc ::profile-id profile-id)
|
||||
(assoc ::ignore-index-errors? true))]
|
||||
(with-meta
|
||||
(sse/response #(import-binfile params))
|
||||
{::audit/props {:file nil}})))
|
||||
|
||||
(defn- import-binfile
|
||||
[{:keys [::wrk/executor ::project-id] :as params}]
|
||||
(db/tx-run! params
|
||||
(fn [{:keys [::db/conn] :as params}]
|
||||
;; NOTE: the importation process performs some operations that
|
||||
;; are not very friendly with virtual threads, and for avoid
|
||||
;; unexpected blocking of other concurrent operations we
|
||||
;; dispatch that operation to a dedicated executor.
|
||||
(let [result (p/thread-call executor (partial import! params))]
|
||||
(db/update! conn :project
|
||||
{:modified-at (dt/now)}
|
||||
{:id project-id})
|
||||
(deref result)))))
|
||||
(db/with-atomic [conn pool]
|
||||
(projects/check-read-permissions! conn profile-id project-id)
|
||||
(let [ids (import! (assoc cfg
|
||||
::input (:path file)
|
||||
::project-id project-id
|
||||
::ignore-index-errors? true))]
|
||||
(rph/with-meta ids
|
||||
{::audit/props {:file nil :file-ids ids}}))))
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
@@ -44,17 +43,15 @@
|
||||
|
||||
(defn- get-file
|
||||
"A specialized version of get-file for comments module."
|
||||
[{:keys [::db/conn] :as cfg} file-id page-id]
|
||||
(if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id])
|
||||
(files/decode-row))]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
|
||||
[conn file-id page-id]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn file-id)]
|
||||
(if-let [{:keys [data] :as file} (some-> (db/exec-one! conn [sql:get-file file-id]) (files/decode-row))]
|
||||
(-> file
|
||||
(assoc :page-name (dm/get-in data [:pages-index page-id :name]))
|
||||
(assoc :page-id page-id)))
|
||||
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "file not found")))
|
||||
(assoc :page-id page-id))
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "file not found"))))
|
||||
|
||||
(defn- get-comment-thread
|
||||
[conn thread-id & {:as opts}]
|
||||
@@ -291,37 +288,38 @@
|
||||
(sv/defmethod ::create-comment-thread
|
||||
{::doc/added "1.15"
|
||||
::webhooks/event? true}
|
||||
[cfg {:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
(let [{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
|
||||
[{:keys [::db/pool] :as cfg}
|
||||
{:keys [::rpc/profile-id ::rpc/request-at file-id page-id share-id position content frame-id]}]
|
||||
|
||||
(run! (partial quotes/check-quote! conn)
|
||||
(list {::quotes/id ::quotes/comment-threads-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id file-id}
|
||||
{::quotes/id ::quotes/comments-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id file-id}))
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
|
||||
(rtry/with-retry {::rtry/when rtry/conflict-exception?
|
||||
::rtry/max-retries 3
|
||||
::rtry/label "create-comment-thread"
|
||||
::db/conn conn}
|
||||
(create-comment-thread conn
|
||||
{:created-at request-at
|
||||
:profile-id profile-id
|
||||
:file-id file-id
|
||||
:page-id page-id
|
||||
:page-name page-name
|
||||
:position position
|
||||
:content content
|
||||
:frame-id frame-id}))))))
|
||||
(run! (partial quotes/check-quote! conn)
|
||||
(list {::quotes/id ::quotes/comment-threads-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id file-id}
|
||||
{::quotes/id ::quotes/comments-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id file-id}))
|
||||
|
||||
(rtry/with-retry {::rtry/when rtry/conflict-exception?
|
||||
::rtry/max-retries 3
|
||||
::rtry/label "create-comment-thread"
|
||||
::db/conn conn}
|
||||
(create-comment-thread conn
|
||||
{:created-at request-at
|
||||
:profile-id profile-id
|
||||
:file-id file-id
|
||||
:page-id page-id
|
||||
:page-name page-name
|
||||
:position position
|
||||
:content content
|
||||
:frame-id frame-id})))))
|
||||
|
||||
|
||||
(defn- create-comment-thread
|
||||
@@ -404,7 +402,8 @@
|
||||
|
||||
;; --- COMMAND: Add Comment
|
||||
|
||||
(declare ^:private get-comment-thread)
|
||||
(declare get-comment-thread)
|
||||
(declare create-comment)
|
||||
|
||||
(s/def ::create-comment
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
@@ -414,51 +413,49 @@
|
||||
(sv/defmethod ::create-comment
|
||||
{::doc/added "1.15"
|
||||
::webhooks/event? true}
|
||||
[cfg {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content]}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
|
||||
{:keys [team-id project-id page-name] :as file} (get-file cfg file-id page-id)]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at thread-id share-id content] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)
|
||||
{:keys [team-id project-id page-name] :as file} (get-file conn file-id page-id)]
|
||||
|
||||
(files/check-comment-permissions! conn profile-id (:id file) share-id)
|
||||
(quotes/check-quote! conn
|
||||
{::quotes/id ::quotes/comments-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id (:id file)})
|
||||
(files/check-comment-permissions! conn profile-id (:id file) share-id)
|
||||
(quotes/check-quote! conn
|
||||
{::quotes/id ::quotes/comments-per-file
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id
|
||||
::quotes/project-id project-id
|
||||
::quotes/file-id (:id file)})
|
||||
|
||||
;; Update the page-name cached attribute on comment thread table.
|
||||
(when (not= page-name (:page-name thread))
|
||||
(db/update! conn :comment-thread
|
||||
{:page-name page-name}
|
||||
{:id thread-id}))
|
||||
;; Update the page-name cached attribute on comment thread table.
|
||||
(when (not= page-name (:page-name thread))
|
||||
(db/update! conn :comment-thread
|
||||
{:page-name page-name}
|
||||
{:id thread-id}))
|
||||
|
||||
(let [comment (db/insert! conn :comment
|
||||
{:id (uuid/next)
|
||||
:created-at request-at
|
||||
:modified-at request-at
|
||||
:thread-id thread-id
|
||||
:owner-id profile-id
|
||||
:content content})
|
||||
props {:file-id file-id
|
||||
:share-id nil}]
|
||||
(let [comment (db/insert! conn :comment
|
||||
{:id (uuid/next)
|
||||
:created-at request-at
|
||||
:modified-at request-at
|
||||
:thread-id thread-id
|
||||
:owner-id profile-id
|
||||
:content content})
|
||||
props {:file-id file-id
|
||||
:share-id nil}]
|
||||
|
||||
;; Update thread modified-at attribute and assoc the current
|
||||
;; profile to the participant set.
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
:participants (-> (:participants thread #{})
|
||||
(conj profile-id)
|
||||
(db/tjson))}
|
||||
{:id thread-id})
|
||||
;; Update thread modified-at attribute and assoc the current
|
||||
;; profile to the participant set.
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
:participants (-> (:participants thread #{})
|
||||
(conj profile-id)
|
||||
(db/tjson))}
|
||||
{:id thread-id})
|
||||
|
||||
;; Update the current profile status in relation to the
|
||||
;; current thread.
|
||||
(upsert-comment-thread-status! conn profile-id thread-id request-at)
|
||||
|
||||
(vary-meta comment assoc ::audit/props props))))))
|
||||
;; Update the current profile status in relation to the
|
||||
;; current thread.
|
||||
(upsert-comment-thread-status! conn profile-id thread-id request-at)
|
||||
|
||||
(vary-meta comment assoc ::audit/props props)))))
|
||||
|
||||
;; --- COMMAND: Update Comment
|
||||
|
||||
@@ -469,31 +466,29 @@
|
||||
|
||||
(sv/defmethod ::update-comment
|
||||
{::doc/added "1.15"}
|
||||
[cfg {:keys [::rpc/profile-id ::rpc/request-at id share-id content]}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id share-id content] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [thread-id] :as comment} (get-comment conn id ::db/for-update? true)
|
||||
{:keys [file-id page-id owner-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
|
||||
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(let [{:keys [thread-id owner-id] :as comment} (get-comment conn id ::db/for-update? true)
|
||||
{:keys [file-id page-id] :as thread} (get-comment-thread conn thread-id ::db/for-update? true)]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
;; Don't allow edit comments to not owners
|
||||
(when-not (= owner-id profile-id)
|
||||
(ex/raise :type :validation
|
||||
:code :not-allowed))
|
||||
|
||||
;; Don't allow edit comments to not owners
|
||||
(when-not (= owner-id profile-id)
|
||||
(ex/raise :type :validation
|
||||
:code :not-allowed))
|
||||
(let [{:keys [page-name] :as file} (get-file conn file-id page-id)]
|
||||
(db/update! conn :comment
|
||||
{:content content
|
||||
:modified-at request-at}
|
||||
{:id id})
|
||||
|
||||
(let [{:keys [page-name] :as file} (get-file cfg file-id page-id)]
|
||||
(db/update! conn :comment
|
||||
{:content content
|
||||
:modified-at request-at}
|
||||
{:id id})
|
||||
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
:page-name page-name}
|
||||
{:id thread-id})
|
||||
nil)))))
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
:page-name page-name}
|
||||
{:id thread-id})
|
||||
nil))))
|
||||
|
||||
;; --- COMMAND: Delete Comment Thread
|
||||
|
||||
@@ -504,7 +499,7 @@
|
||||
|
||||
(sv/defmethod ::delete-comment-thread
|
||||
{::doc/added "1.15"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id]}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id share-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [owner-id file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
@@ -544,12 +539,12 @@
|
||||
|
||||
(sv/defmethod ::update-comment-thread-position
|
||||
{::doc/added "1.15"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id position frame-id share-id]}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id position frame-id share-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
{:modified-at (::rpc/request-at params)
|
||||
:position (db/pgpoint position)
|
||||
:frame-id frame-id}
|
||||
{:id (:id thread)})
|
||||
@@ -564,12 +559,12 @@
|
||||
|
||||
(sv/defmethod ::update-comment-thread-frame
|
||||
{::doc/added "1.15"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id ::rpc/request-at id frame-id share-id]}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id frame-id share-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [{:keys [file-id] :as thread} (get-comment-thread conn id ::db/for-update? true)]
|
||||
(files/check-comment-permissions! conn profile-id file-id share-id)
|
||||
(db/update! conn :comment-thread
|
||||
{:modified-at request-at
|
||||
{:modified-at (::rpc/request-at params)
|
||||
:frame-id frame-id}
|
||||
{:id id})
|
||||
nil)))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,20 +7,15 @@
|
||||
(ns app.rpc.commands.files-create
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.files.features :as ffeat]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.projects :as projects]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.permissions :as perms]
|
||||
[app.rpc.quotes :as quotes]
|
||||
@@ -29,7 +24,7 @@
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[clojure.set :as set]))
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
(defn create-file-role!
|
||||
[conn {:keys [file-id profile-id role]}]
|
||||
@@ -39,34 +34,27 @@
|
||||
(db/insert! conn :file-profile-rel))))
|
||||
|
||||
(defn create-file
|
||||
[{:keys [::db/conn] :as cfg}
|
||||
{:keys [id name project-id is-shared revn
|
||||
modified-at deleted-at create-page
|
||||
ignore-sync-until features]
|
||||
:or {is-shared false revn 0 create-page true}
|
||||
:as params}]
|
||||
|
||||
(dm/assert!
|
||||
"expected a valid connection"
|
||||
(db/connection? conn))
|
||||
[conn {:keys [id name project-id is-shared revn
|
||||
modified-at deleted-at create-page
|
||||
ignore-sync-until features]
|
||||
:or {is-shared false revn 0 create-page true}
|
||||
:as params}]
|
||||
|
||||
(let [id (or id (uuid/next))
|
||||
features (->> features
|
||||
(into (files/get-default-features))
|
||||
(files/check-features-compatibility!))
|
||||
|
||||
pointers (pmap/create-tracked)
|
||||
pmap? (contains? features "fdata/pointer-map")
|
||||
omap? (contains? features "fdata/objects-map")
|
||||
|
||||
pointers (atom {})
|
||||
data (binding [pmap/*tracked* pointers
|
||||
cfeat/*current* features
|
||||
cfeat/*wrap-with-objects-map-fn* (if omap? omap/wrap identity)
|
||||
cfeat/*wrap-with-pointer-map-fn* (if pmap? pmap/wrap identity)]
|
||||
ffeat/*current* features
|
||||
ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity)
|
||||
ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)]
|
||||
(if create-page
|
||||
(ctf/make-file-data id)
|
||||
(ctf/make-file-data id nil)))
|
||||
|
||||
features (->> (set/difference features cfeat/frontend-only-features)
|
||||
(db/create-array conn "text"))
|
||||
|
||||
features (db/create-array conn "text" features)
|
||||
file (db/insert! conn :file
|
||||
(d/without-nils
|
||||
{:id id
|
||||
@@ -81,69 +69,40 @@
|
||||
:deleted-at deleted-at}))]
|
||||
|
||||
(binding [pmap/*tracked* pointers]
|
||||
(feat.fdata/persist-pointers! cfg id))
|
||||
(files/persist-pointers! conn id))
|
||||
|
||||
(->> (assoc params :file-id id :role :owner)
|
||||
(create-file-role! conn))
|
||||
|
||||
(db/update! conn :project
|
||||
{:modified-at (dt/now)}
|
||||
{:id project-id})
|
||||
{:modified-at (dt/now)}
|
||||
{:id project-id})
|
||||
|
||||
(files/decode-row file)))
|
||||
|
||||
(def ^:private schema:create-file
|
||||
[:map {:title "create-file"}
|
||||
[:name :string]
|
||||
[:project-id ::sm/uuid]
|
||||
[:id {:optional true} ::sm/uuid]
|
||||
[:is-shared {:optional true} :boolean]
|
||||
[:features {:optional true} ::cfeat/features]])
|
||||
(s/def ::create-file
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::files/name
|
||||
::files/project-id]
|
||||
:opt-un [::files/id
|
||||
::files/is-shared
|
||||
::files/features]))
|
||||
|
||||
(sv/defmethod ::create-file
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files
|
||||
::webhooks/event? true
|
||||
::sm/params schema:create-file}
|
||||
[cfg {:keys [::rpc/profile-id project-id] :as params}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(projects/check-edition-permissions! conn profile-id project-id)
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:project-id project-id)
|
||||
team-id (:id team)
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(projects/check-edition-permissions! conn profile-id project-id)
|
||||
(let [team-id (files/get-team-id conn project-id)
|
||||
params (assoc params :profile-id profile-id)]
|
||||
|
||||
;; When we create files, we only need to respect the team
|
||||
;; features, because some features can be enabled
|
||||
;; globally, but the team is still not migrated properly.
|
||||
features (-> (cfeat/get-team-enabled-features cf/flags team)
|
||||
(cfeat/check-client-features! (:features params)))
|
||||
(run! (partial quotes/check-quote! conn)
|
||||
(list {::quotes/id ::quotes/files-per-project
|
||||
::quotes/team-id team-id
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/project-id project-id}))
|
||||
|
||||
;; We also include all no migration features declared by
|
||||
;; client; that enables the ability to enable a runtime
|
||||
;; feature on frontend and make it permanent on file
|
||||
features (-> (:features params #{})
|
||||
(set/intersection cfeat/no-migration-features)
|
||||
(set/union features))
|
||||
|
||||
params (-> params
|
||||
(assoc :profile-id profile-id)
|
||||
(assoc :features features))]
|
||||
|
||||
(run! (partial quotes/check-quote! conn)
|
||||
(list {::quotes/id ::quotes/files-per-project
|
||||
::quotes/team-id team-id
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/project-id project-id}))
|
||||
|
||||
;; When newly computed features does not match exactly with
|
||||
;; the features defined on team row, we update it.
|
||||
(when (not= features (:features team))
|
||||
(let [features (db/create-array conn "text" features)]
|
||||
(db/update! conn :team
|
||||
{:features features}
|
||||
{:id team-id})))
|
||||
|
||||
(-> (create-file cfg params)
|
||||
(vary-meta assoc ::audit/props {:team-id team-id}))))))
|
||||
(-> (create-file conn params)
|
||||
(vary-meta assoc ::audit/props {:team-id team-id})))))
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.rpc.commands.files-snapshot
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.main :as-alias main]
|
||||
[app.media :as media]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.storage :as sto]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]))
|
||||
|
||||
(defn check-authorized!
|
||||
[{:keys [::db/pool]} profile-id]
|
||||
(when-not (or (= "devenv" (cf/get :host))
|
||||
(let [profile (ex/ignoring (profile/get-profile pool profile-id))
|
||||
admins (or (cf/get :admins) #{})]
|
||||
(contains? admins (:email profile))))
|
||||
(ex/raise :type :authentication
|
||||
:code :authentication-required
|
||||
:hint "only admins allowed")))
|
||||
|
||||
(defn get-file-snapshots
|
||||
[{:keys [::db/conn]} {:keys [file-id limit start-at]
|
||||
:or {limit Long/MAX_VALUE}}]
|
||||
(let [query (str "select id, label, revn, created_at "
|
||||
" from file_change "
|
||||
" where file_id = ? "
|
||||
" and created_at < ? "
|
||||
" and data is not null "
|
||||
" order by created_at desc "
|
||||
" limit ?")
|
||||
start-at (or start-at (dt/now))
|
||||
limit (min limit 20)]
|
||||
|
||||
(->> (db/exec! conn [query file-id start-at limit])
|
||||
(mapv (fn [row]
|
||||
(update row :created-at dt/format-instant :rfc1123))))))
|
||||
|
||||
(def ^:private schema:get-file-snapshots
|
||||
[:map [:file-id ::sm/uuid]])
|
||||
|
||||
(sv/defmethod ::get-file-snapshots
|
||||
{::doc/added "1.20"
|
||||
::doc/skip true
|
||||
::sm/params schema:get-file-snapshots}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(check-authorized! cfg profile-id)
|
||||
(db/run! cfg #(get-file-snapshots % params)))
|
||||
|
||||
(defn restore-file-snapshot!
|
||||
[{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id id]}]
|
||||
(let [storage (media/configure-assets-storage storage conn)
|
||||
params {:id id :file-id file-id}
|
||||
options {:columns [:id :data :revn]}
|
||||
snapshot (db/get* conn :file-change params options)]
|
||||
|
||||
(when (and (some? snapshot)
|
||||
(some? (:data snapshot)))
|
||||
|
||||
(l/debug :hint "snapshot found"
|
||||
:snapshot-id (:id snapshot)
|
||||
:file-id file-id)
|
||||
|
||||
(db/update! conn :file
|
||||
{:data (:data snapshot)}
|
||||
{:id file-id})
|
||||
|
||||
;; clean object thumbnails
|
||||
(let [sql (str "delete from file_object_thumbnail "
|
||||
" where file_id=? returning media_id")
|
||||
res (db/exec! conn [sql file-id])]
|
||||
|
||||
(doseq [media-id (into #{} (keep :media-id) res)]
|
||||
(sto/del-object! storage media-id)))
|
||||
|
||||
;; clean object thumbnails
|
||||
(let [sql (str "delete from file_thumbnail "
|
||||
" where file_id=? returning media_id")
|
||||
res (db/exec! conn [sql file-id])]
|
||||
(doseq [media-id (into #{} (keep :media-id) res)]
|
||||
(sto/del-object! storage media-id)))
|
||||
|
||||
{:id (:id snapshot)})))
|
||||
|
||||
(def ^:private schema:restore-file-snapshot
|
||||
[:map
|
||||
[:file-id ::sm/uuid]
|
||||
[:id ::sm/uuid]])
|
||||
|
||||
(sv/defmethod ::restore-file-snapshot
|
||||
{::doc/added "1.20"
|
||||
::doc/skip true
|
||||
::sm/params schema:restore-file-snapshot}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(check-authorized! cfg profile-id)
|
||||
(db/tx-run! cfg #(restore-file-snapshot! % params)))
|
||||
|
||||
(defn take-file-snapshot!
|
||||
[{:keys [::db/conn]} {:keys [file-id label]}]
|
||||
(when-let [file (db/get* conn :file {:id file-id})]
|
||||
(let [id (uuid/next)
|
||||
label (or label (str "Snapshot at " (dt/format-instant (dt/now) :rfc1123)))]
|
||||
(l/debug :hint "persisting file snapshot" :file-id file-id :label label)
|
||||
(db/insert! conn :file-change
|
||||
{:id id
|
||||
:revn (:revn file)
|
||||
:data (:data file)
|
||||
:features (:features file)
|
||||
:file-id (:id file)
|
||||
:label label})
|
||||
{:id id})))
|
||||
|
||||
(def ^:private schema:take-file-snapshot
|
||||
[:map [:file-id ::sm/uuid]])
|
||||
|
||||
(sv/defmethod ::take-file-snapshot
|
||||
{::doc/added "1.20"
|
||||
::doc/skip true
|
||||
::sm/params schema:take-file-snapshot}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(check-authorized! cfg profile-id)
|
||||
(db/tx-run! cfg #(take-file-snapshot! % params)))
|
||||
|
||||
@@ -7,26 +7,21 @@
|
||||
(ns app.rpc.commands.files-temp
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.changes :as fch]
|
||||
[app.common.pages :as cp]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.files-create :as files.create]
|
||||
[app.rpc.commands.files-create :refer [create-file]]
|
||||
[app.rpc.commands.files-update :as-alias files.update]
|
||||
[app.rpc.commands.projects :as projects]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[clojure.set :as set]
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
|
||||
;; --- MUTATION COMMAND: create-temp-file
|
||||
|
||||
(s/def ::create-page ::us/boolean)
|
||||
@@ -43,35 +38,25 @@
|
||||
(sv/defmethod ::create-temp-file
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files}
|
||||
[cfg {:keys [::rpc/profile-id project-id] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(projects/check-edition-permissions! conn profile-id project-id)
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:project-id project-id)
|
||||
|
||||
;; When we create files, we only need to respect the team
|
||||
;; features, because some features can be enabled
|
||||
;; globally, but the team is still not migrated properly.
|
||||
features (-> (cfeat/get-team-enabled-features cf/flags team)
|
||||
(cfeat/check-client-features! (:features params)))
|
||||
|
||||
;; We also include all no migration features declared by
|
||||
;; client; that enables the ability to enable a runtime
|
||||
;; feature on frontend and make it permanent on file
|
||||
features (-> (:features params #{})
|
||||
(set/intersection cfeat/no-migration-features)
|
||||
(set/union features))
|
||||
|
||||
params (-> params
|
||||
(assoc :profile-id profile-id)
|
||||
(assoc :deleted-at (dt/in-future {:days 1}))
|
||||
(assoc :features features))]
|
||||
|
||||
(files.create/create-file cfg params)))))
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(projects/check-edition-permissions! conn profile-id project-id)
|
||||
(create-file conn (assoc params :profile-id profile-id :deleted-at (dt/in-future {:days 1})))))
|
||||
|
||||
;; --- MUTATION COMMAND: update-temp-file
|
||||
|
||||
(defn update-temp-file
|
||||
[conn {:keys [profile-id session-id id revn changes] :as params}]
|
||||
(db/insert! conn :file-change
|
||||
{:id (uuid/next)
|
||||
:session-id session-id
|
||||
:profile-id profile-id
|
||||
:created-at (dt/now)
|
||||
:file-id id
|
||||
:revn revn
|
||||
:data nil
|
||||
:changes (blob/encode changes)}))
|
||||
|
||||
(s/def ::update-temp-file
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::files.update/changes
|
||||
@@ -82,18 +67,10 @@
|
||||
(sv/defmethod ::update-temp-file
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files}
|
||||
[cfg {:keys [::rpc/profile-id session-id id revn changes] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn]}]
|
||||
(db/insert! conn :file-change
|
||||
{:id (uuid/next)
|
||||
:session-id session-id
|
||||
:profile-id profile-id
|
||||
:created-at (dt/now)
|
||||
:file-id id
|
||||
:revn revn
|
||||
:data nil
|
||||
:changes (blob/encode changes)})
|
||||
nil)))
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(update-temp-file conn (assoc params :profile-id profile-id))
|
||||
nil))
|
||||
|
||||
;; --- MUTATION COMMAND: persist-temp-file
|
||||
|
||||
@@ -109,16 +86,16 @@
|
||||
(ex/raise :type :validation
|
||||
:code :cant-persist-already-persisted-file))
|
||||
|
||||
|
||||
(let [data
|
||||
(->> revs
|
||||
(mapcat #(->> % :changes blob/decode))
|
||||
(fch/process-changes (blob/decode (:data file))))]
|
||||
(db/update! conn :file
|
||||
{:deleted-at nil
|
||||
:revn revn
|
||||
:data (blob/encode data)}
|
||||
{:id id}))
|
||||
(loop [revs (seq revs)
|
||||
data (blob/decode (:data file))]
|
||||
(if-let [rev (first revs)]
|
||||
(recur (rest revs)
|
||||
(->> rev :changes blob/decode (cp/process-changes data)))
|
||||
(db/update! conn :file
|
||||
{:deleted-at nil
|
||||
:revn revn
|
||||
:data (blob/encode data)}
|
||||
{:id id})))
|
||||
nil))
|
||||
|
||||
(s/def ::persist-temp-file
|
||||
@@ -128,7 +105,7 @@
|
||||
(sv/defmethod ::persist-temp-file
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files}
|
||||
[cfg {:keys [::rpc/profile-id id] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn]}]
|
||||
(files/check-edition-permissions! conn profile-id id)
|
||||
(persist-temp-file conn params))))
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id id)
|
||||
(persist-temp-file conn params)))
|
||||
|
||||
@@ -8,25 +8,23 @@
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.helpers :as cfh]
|
||||
[app.common.files.migrations :as fmg]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.geom.shapes :as gsh]
|
||||
[app.common.pages.helpers :as cph]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.thumbnails :as thc]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.shape-tree :as ctt]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.db.sql :as sql]
|
||||
[app.loggers.audit :as-alias audit]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.media :as media]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.climit :as-alias climit]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.cond :as-alias cond]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.storage :as sto]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
@@ -41,74 +39,117 @@
|
||||
|
||||
;; --- COMMAND QUERY: get-file-object-thumbnails
|
||||
|
||||
(defn- get-object-thumbnails-by-tag
|
||||
[conn file-id tag]
|
||||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
" where file_id=? and tag=?")
|
||||
res (db/exec! conn [sql file-id tag])]
|
||||
(->> res
|
||||
(d/index-by :object-id (fn [row]
|
||||
(files/resolve-public-uri (:media-id row))))
|
||||
(d/without-nils))))
|
||||
(defn- get-public-uri
|
||||
[media-id]
|
||||
(str (cf/get :public-uri) "/assets/by-id/" media-id))
|
||||
|
||||
(defn- get-object-thumbnails
|
||||
([conn file-id]
|
||||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
"select object_id, data, media_id "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=?")
|
||||
res (db/exec! conn [sql file-id])]
|
||||
(->> res
|
||||
(d/index-by :object-id (fn [row]
|
||||
(files/resolve-public-uri (:media-id row))))
|
||||
(or (some-> row :media-id get-public-uri)
|
||||
(:data row))))
|
||||
(d/without-nils))))
|
||||
|
||||
([conn file-id object-ids]
|
||||
(let [sql (str/concat
|
||||
"select object_id, media_id, tag "
|
||||
" from file_tagged_object_thumbnail"
|
||||
"select object_id, data, media_id "
|
||||
" from file_object_thumbnail"
|
||||
" where file_id=? and object_id = ANY(?)")
|
||||
ids (db/create-array conn "text" (seq object-ids))
|
||||
res (db/exec! conn [sql file-id ids])]
|
||||
|
||||
(->> res
|
||||
(d/index-by :object-id (fn [row]
|
||||
(files/resolve-public-uri (:media-id row))))
|
||||
(d/without-nils)))))
|
||||
(d/index-by :object-id
|
||||
(fn [row]
|
||||
(or (some-> row :media-id get-public-uri)
|
||||
(:data row)))
|
||||
res))))
|
||||
|
||||
(sv/defmethod ::get-file-object-thumbnails
|
||||
"Retrieve a file object thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files
|
||||
::sm/params [:map {:title "get-file-object-thumbnails"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:tag {:optional true} :string]]
|
||||
[:file-id ::sm/uuid]]
|
||||
::sm/result [:map-of :string :string]
|
||||
::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
|
||||
::cond/reuse-key? true
|
||||
::cond/key-fn files/get-file-etag}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id tag] :as params}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
(if tag
|
||||
(get-object-thumbnails-by-tag conn file-id tag)
|
||||
(get-object-thumbnails conn file-id))))
|
||||
(get-object-thumbnails conn file-id)))
|
||||
|
||||
;; --- COMMAND QUERY: get-file-thumbnail
|
||||
|
||||
;; FIXME: refactor to support uploading data to storage
|
||||
|
||||
(defn get-file-thumbnail
|
||||
[conn file-id revn]
|
||||
(let [sql (sql/select :file-thumbnail
|
||||
(cond-> {:file-id file-id}
|
||||
revn (assoc :revn revn))
|
||||
{:limit 1
|
||||
:order-by [[:revn :desc]]})
|
||||
row (db/exec-one! conn sql)]
|
||||
(when-not row
|
||||
(ex/raise :type :not-found
|
||||
:code :file-thumbnail-not-found))
|
||||
|
||||
{:data (:data row)
|
||||
:props (some-> (:props row) db/decode-transit-pgobject)
|
||||
:revn (:revn row)
|
||||
:file-id (:file-id row)}))
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::file-id ::us/uuid)
|
||||
|
||||
(s/def ::get-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::revn]))
|
||||
|
||||
(sv/defmethod ::get-file-thumbnail
|
||||
"Method used in frontend for obtain the file thumbnail (used in the
|
||||
dashboard)."
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id file-id revn]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
(-> (get-file-thumbnail conn file-id revn)
|
||||
(rph/with-http-cache long-cache-duration))))
|
||||
|
||||
|
||||
;; --- COMMAND QUERY: get-file-data-for-thumbnail
|
||||
|
||||
;; FIXME: performance issue, handle new media_id
|
||||
;;
|
||||
;; We need to improve how we set frame for thumbnail in order to avoid
|
||||
;; loading all pages into memory for find the frame set for thumbnail.
|
||||
|
||||
(defn get-file-data-for-thumbnail
|
||||
[{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
|
||||
[conn {:keys [data id] :as file}]
|
||||
(letfn [;; function responsible on finding the frame marked to be
|
||||
;; used as thumbnail; the returned frame always have
|
||||
;; the :page-id set to the page that it belongs.
|
||||
(get-thumbnail-frame [{:keys [data]}]
|
||||
(d/seek #(or (:use-for-thumbnail %)
|
||||
(:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21)
|
||||
|
||||
(get-thumbnail-frame [data]
|
||||
;; NOTE: this is a hack for avoid perform blocking
|
||||
;; operation inside the for loop, clojure lazy-seq uses
|
||||
;; synchronized blocks that does not plays well with
|
||||
;; virtual threads, so we need to perform the load
|
||||
;; operation first. This operation forces all pointer maps
|
||||
;; load into the memory.
|
||||
(->> (-> data :pages-index vals)
|
||||
(filter pmap/pointer-map?)
|
||||
(run! pmap/load!))
|
||||
|
||||
;; Then proceed to find the frame set for thumbnail
|
||||
|
||||
(d/seek :use-for-thumbnail?
|
||||
(for [page (-> data :pages-index vals)
|
||||
frame (-> page :objects ctt/get-frames)]
|
||||
(assoc frame :page-id (:id page)))))
|
||||
@@ -117,28 +158,28 @@
|
||||
;; all unneeded shapes if a concrete frame is provided. If no
|
||||
;; frame, the objects is returned untouched.
|
||||
(filter-objects [objects frame-id]
|
||||
(d/index-by :id (cfh/get-children-with-self objects frame-id)))
|
||||
(d/index-by :id (cph/get-children-with-self objects frame-id)))
|
||||
|
||||
;; function responsible of assoc available thumbnails
|
||||
;; to frames and remove all children shapes from objects if
|
||||
;; thumbnails is available
|
||||
(assoc-thumbnails [objects page-id thumbnails]
|
||||
(loop [objects objects
|
||||
frames (filter cfh/frame-shape? (vals objects))]
|
||||
frames (filter cph/frame-shape? (vals objects))]
|
||||
|
||||
(if-let [frame (-> frames first)]
|
||||
(let [frame-id (:id frame)
|
||||
object-id (thc/fmt-object-id (:id file) page-id frame-id "frame")
|
||||
frame (if-let [thumb (get thumbnails object-id)]
|
||||
(assoc frame :thumbnail thumb :shapes [])
|
||||
(dissoc frame :thumbnail))
|
||||
(let [frame-id (:id frame)
|
||||
object-id (str page-id frame-id)
|
||||
frame (if-let [thumb (get thumbnails object-id)]
|
||||
(assoc frame :thumbnail thumb :shapes [])
|
||||
(dissoc frame :thumbnail))
|
||||
|
||||
children-ids
|
||||
(cfh/get-children-ids objects frame-id)
|
||||
(cph/get-children-ids objects frame-id)
|
||||
|
||||
bounds
|
||||
(when (:show-content frame)
|
||||
(gsh/shapes->rect (cons frame (map (d/getf objects) children-ids))))
|
||||
(gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects))))))
|
||||
|
||||
frame
|
||||
(cond-> frame
|
||||
@@ -155,88 +196,104 @@
|
||||
|
||||
objects)))]
|
||||
|
||||
(let [frame (get-thumbnail-frame file)
|
||||
frame-id (:id frame)
|
||||
page-id (or (:page-id frame)
|
||||
(-> data :pages first))
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(let [frame (get-thumbnail-frame data)
|
||||
frame-id (:id frame)
|
||||
page-id (or (:page-id frame)
|
||||
(-> data :pages first))
|
||||
|
||||
page (dm/get-in data [:pages-index page-id])
|
||||
page (cond-> page (pmap/pointer-map? page) deref)
|
||||
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
||||
page (dm/get-in data [:pages-index page-id])
|
||||
page (cond-> page (pmap/pointer-map? page) deref)
|
||||
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
||||
|
||||
obj-ids (map #(thc/fmt-object-id (:id file) page-id % "frame") frame-ids)
|
||||
thumbs (get-object-thumbnails conn id obj-ids)]
|
||||
obj-ids (map #(str page-id %) frame-ids)
|
||||
thumbs (get-object-thumbnails conn id obj-ids)]
|
||||
|
||||
(cond-> page
|
||||
;; If we have frame, we need to specify it on the page level
|
||||
;; and remove the all other unrelated objects.
|
||||
(some? frame-id)
|
||||
(-> (assoc :thumbnail-frame-id frame-id)
|
||||
(update :objects filter-objects frame-id))
|
||||
(cond-> page
|
||||
;; If we have frame, we need to specify it on the page level
|
||||
;; and remove the all other unrelated objects.
|
||||
(some? frame-id)
|
||||
(-> (assoc :thumbnail-frame-id frame-id)
|
||||
(update :objects filter-objects frame-id))
|
||||
|
||||
;; Assoc the available thumbnails and prune not visible shapes
|
||||
;; for avoid transfer unnecessary data.
|
||||
:always
|
||||
(update :objects assoc-thumbnails page-id thumbs)))))
|
||||
|
||||
(def ^:private
|
||||
schema:get-file-data-for-thumbnail
|
||||
(sm/define
|
||||
[:map {:title "get-file-data-for-thumbnail"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:features {:optional true} ::cfeat/features]]))
|
||||
|
||||
(def ^:private
|
||||
schema:partial-file
|
||||
(sm/define
|
||||
[:map {:title "PartialFile"}
|
||||
[:id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:page :any]]))
|
||||
;; Assoc the available thumbnails and prune not visible shapes
|
||||
;; for avoid transfer unnecessary data.
|
||||
:always
|
||||
(update :objects assoc-thumbnails page-id thumbs))))))
|
||||
|
||||
(sv/defmethod ::get-file-data-for-thumbnail
|
||||
"Retrieves the data for generate the thumbnail of the file. Used
|
||||
mainly for render thumbnails on dashboard."
|
||||
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files
|
||||
::sm/params schema:get-file-data-for-thumbnail
|
||||
::sm/result schema:partial-file}
|
||||
[cfg {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
|
||||
(let [team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:file-id file-id)
|
||||
|
||||
file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)]
|
||||
(-> (files/get-file cfg file-id :migrate? false)
|
||||
(update :data feat.fdata/process-pointers deref)
|
||||
(fmg/migrate-file)))]
|
||||
|
||||
(-> (cfeat/get-team-enabled-features cf/flags team)
|
||||
(cfeat/check-client-features! (:features params))
|
||||
(cfeat/check-file-features! (:features file) (:features params)))
|
||||
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:page (get-file-data-for-thumbnail cfg file)}))))
|
||||
::sm/params [:map {:title "get-file-data-for-thumbnail"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:features {:optional true} ::files/features]]
|
||||
::sm/result [:map {:title "PartialFile"}
|
||||
[:id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:page :any]]}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id features] :as props}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
;; NOTE: we force here the "storage/pointer-map" feature, because
|
||||
;; it used internally only and is independent if user supports it
|
||||
;; or not.
|
||||
(let [feat (into #{"storage/pointer-map"} features)
|
||||
file (files/get-file conn file-id feat)]
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:page (get-file-data-for-thumbnail conn file)})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; MUTATION COMMANDS
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-object-thumbnail
|
||||
|
||||
(def sql:upsert-object-thumbnail
|
||||
"insert into file_object_thumbnail(file_id, object_id, data)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set data = ?;")
|
||||
|
||||
(defn upsert-file-object-thumbnail!
|
||||
[conn {:keys [file-id object-id data]}]
|
||||
(if data
|
||||
(db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data])
|
||||
(db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})))
|
||||
|
||||
(s/def ::data (s/nilable ::us/string))
|
||||
(s/def ::object-id ::us/string)
|
||||
|
||||
(s/def ::upsert-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::object-id]
|
||||
:opt-un [::data]))
|
||||
|
||||
(sv/defmethod ::upsert-file-object-thumbnail
|
||||
{::doc/added "1.17"
|
||||
::doc/deprecated "1.19"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
|
||||
(when-not (db/read-only? conn)
|
||||
(upsert-file-object-thumbnail! conn params)
|
||||
nil)))
|
||||
|
||||
|
||||
;; --- MUTATION COMMAND: create-file-object-thumbnail
|
||||
|
||||
(def ^:private sql:create-object-thumbnail
|
||||
"insert into file_tagged_object_thumbnail(file_id, object_id, media_id, tag)
|
||||
values (?, ?, ?, ?)
|
||||
on conflict(file_id, tag, object_id) do
|
||||
update set media_id = ?
|
||||
returning *;")
|
||||
"insert into file_object_thumbnail(file_id, object_id, media_id)
|
||||
values (?, ?, ?)
|
||||
on conflict(file_id, object_id) do
|
||||
update set media_id = ?;")
|
||||
|
||||
(defn- create-file-object-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} file-id object-id media tag]
|
||||
[{:keys [::db/conn ::sto/storage]} file-id object-id media]
|
||||
|
||||
(let [path (:path media)
|
||||
mtype (:mtype media)
|
||||
@@ -245,30 +302,23 @@
|
||||
(sto/wrap-with-hash hash))
|
||||
media (sto/put-object! storage
|
||||
{::sto/content data
|
||||
::sto/deduplicate? true
|
||||
::sto/touched-at (dt/now)
|
||||
::sto/deduplicate? false
|
||||
:content-type mtype
|
||||
:bucket "file-object-thumbnail"})]
|
||||
|
||||
(db/exec-one! conn [sql:create-object-thumbnail file-id object-id
|
||||
(:id media) tag (:id media)])))
|
||||
(:id media) (:id media)])))
|
||||
|
||||
(def schema:create-file-object-thumbnail
|
||||
[:map {:title "create-file-object-thumbnail"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:object-id :string]
|
||||
[:media ::media/upload]
|
||||
[:tag {:optional true} :string]])
|
||||
|
||||
(s/def ::media (s/nilable ::media/upload))
|
||||
(s/def ::create-file-object-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::object-id ::media]))
|
||||
|
||||
(sv/defmethod ::create-file-object-thumbnail
|
||||
{::doc/added "1.19"
|
||||
::doc/module :files
|
||||
::climit/id :file-thumbnail-ops
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
::audit/skip true
|
||||
::sm/params schema:create-file-object-thumbnail}
|
||||
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media tag]}]
|
||||
{:doc/added "1.19"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id media]}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(media/validate-media-type! media)
|
||||
@@ -278,19 +328,21 @@
|
||||
(-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(create-file-object-thumbnail! file-id object-id media (or tag "frame"))))))
|
||||
(create-file-object-thumbnail! file-id object-id media))
|
||||
nil)))
|
||||
|
||||
;; --- MUTATION COMMAND: delete-file-object-thumbnail
|
||||
|
||||
(defn- delete-file-object-thumbnail!
|
||||
[{:keys [::db/conn ::sto/storage]} file-id object-id]
|
||||
(when-let [{:keys [media-id]} (db/get* conn :file-tagged-object-thumbnail
|
||||
(when-let [{:keys [media-id]} (db/get* conn :file-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id}
|
||||
{::db/for-update? true})]
|
||||
(when media-id
|
||||
(sto/del-object! storage media-id))
|
||||
|
||||
(sto/touch-object! storage media-id)
|
||||
(db/delete! conn :file-tagged-object-thumbnail
|
||||
(db/delete! conn :file-object-thumbnail
|
||||
{:file-id file-id
|
||||
:object-id object-id})
|
||||
nil))
|
||||
@@ -300,10 +352,7 @@
|
||||
:req-un [::file-id ::object-id]))
|
||||
|
||||
(sv/defmethod ::delete-file-object-thumbnail
|
||||
{::doc/added "1.19"
|
||||
::doc/module :files
|
||||
::climit/id :file-thumbnail-ops
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
{:doc/added "1.19"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}]
|
||||
|
||||
@@ -317,6 +366,41 @@
|
||||
(delete-file-object-thumbnail! file-id object-id))
|
||||
nil)))
|
||||
|
||||
;; --- MUTATION COMMAND: upsert-file-thumbnail
|
||||
|
||||
(def ^:private sql:upsert-file-thumbnail
|
||||
"insert into file_thumbnail (file_id, revn, data, props)
|
||||
values (?, ?, ?, ?::jsonb)
|
||||
on conflict(file_id, revn) do
|
||||
update set data = ?, props=?, updated_at=now();")
|
||||
|
||||
(defn- upsert-file-thumbnail!
|
||||
[conn {:keys [file-id revn data props]}]
|
||||
(let [props (db/tjson (or props {}))]
|
||||
(db/exec-one! conn [sql:upsert-file-thumbnail
|
||||
file-id revn data props data props])))
|
||||
|
||||
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::props map?)
|
||||
|
||||
(s/def ::upsert-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::revn ::props ::data]))
|
||||
|
||||
(sv/defmethod ::upsert-file-thumbnail
|
||||
"Creates or updates the file thumbnail. Mainly used for paint the
|
||||
grid thumbnails."
|
||||
{::doc/added "1.17"
|
||||
::doc/deprecated "1.19"
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(upsert-file-thumbnail! conn params)
|
||||
nil)))
|
||||
|
||||
;; --- MUTATION COMMAND: create-file-thumbnail
|
||||
|
||||
(def ^:private sql:create-file-thumbnail
|
||||
@@ -343,29 +427,24 @@
|
||||
:bucket "file-thumbnail"})]
|
||||
(db/exec-one! conn [sql:create-file-thumbnail file-id revn
|
||||
(:id media) props
|
||||
(:id media) props])
|
||||
media))
|
||||
(:id media) props])))
|
||||
|
||||
(s/def ::media ::media/upload)
|
||||
(s/def ::create-file-thumbnail
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id ::revn ::props ::media]))
|
||||
|
||||
(sv/defmethod ::create-file-thumbnail
|
||||
"Creates or updates the file thumbnail. Mainly used for paint the
|
||||
grid thumbnails."
|
||||
{::doc/added "1.19"
|
||||
::doc/module :files
|
||||
::audit/skip true
|
||||
::climit/id :file-thumbnail-ops
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
::sm/params [:map {:title "create-file-thumbnail"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:revn :int]
|
||||
[:media ::media/upload]]}
|
||||
|
||||
::audit/skip true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id file-id)
|
||||
(when-not (db/read-only? conn)
|
||||
(let [media (-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(create-file-thumbnail! params))]
|
||||
|
||||
{:uri (files/resolve-public-uri (:id media))}))))
|
||||
(-> cfg
|
||||
(update ::sto/storage media/configure-assets-storage)
|
||||
(assoc ::db/conn conn)
|
||||
(create-file-thumbnail! params))
|
||||
nil)))
|
||||
|
||||
@@ -6,19 +6,19 @@
|
||||
|
||||
(ns app.rpc.commands.files-update
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.changes :as cpc]
|
||||
[app.common.files.migrations :as fmg]
|
||||
[app.common.files.validate :as val]
|
||||
[app.common.files.features :as ffeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pages :as cp]
|
||||
[app.common.pages.changes :as cpc]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.schema.generators :as smg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.types.file :as ctf]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.http.errors :as errors]
|
||||
[app.loggers.audit :as audit]
|
||||
[app.loggers.webhooks :as webhooks]
|
||||
[app.metrics :as mtx]
|
||||
@@ -26,7 +26,6 @@
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.climit :as climit]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.util.blob :as blob]
|
||||
@@ -34,37 +33,69 @@
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.set :as set]))
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
;; --- SPECS
|
||||
|
||||
(s/def ::changes
|
||||
(s/coll-of map? :kind vector?))
|
||||
|
||||
(s/def ::hint-origin ::us/keyword)
|
||||
(s/def ::hint-events
|
||||
(s/every ::us/keyword :kind vector?))
|
||||
|
||||
(s/def ::change-with-metadata
|
||||
(s/keys :req-un [::changes]
|
||||
:opt-un [::hint-origin
|
||||
::hint-events]))
|
||||
|
||||
(s/def ::changes-with-metadata
|
||||
(s/every ::change-with-metadata :kind vector?))
|
||||
|
||||
(s/def ::session-id ::us/uuid)
|
||||
(s/def ::revn ::us/integer)
|
||||
(s/def ::update-file
|
||||
(s/and
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::files/id ::session-id ::revn]
|
||||
:opt-un [::changes ::changes-with-metadata ::features])
|
||||
(fn [o]
|
||||
(or (contains? o :changes)
|
||||
(contains? o :changes-with-metadata)))))
|
||||
|
||||
|
||||
;; --- SCHEMA
|
||||
|
||||
(def ^:private
|
||||
schema:update-file
|
||||
(sm/define
|
||||
[:map {:title "update-file"}
|
||||
[:id ::sm/uuid]
|
||||
[:session-id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:features {:optional true} ::cfeat/features]
|
||||
[:changes {:optional true} [:vector ::cpc/change]]
|
||||
[:changes-with-metadata {:optional true}
|
||||
[:vector [:map
|
||||
[:changes [:vector ::cpc/change]]
|
||||
[:hint-origin {:optional true} :keyword]
|
||||
[:hint-events {:optional true} [:vector :string]]]]]
|
||||
[:skip-validate {:optional true} :boolean]]))
|
||||
(sm/def! ::changes
|
||||
[:vector ::cpc/change])
|
||||
|
||||
(def ^:private
|
||||
schema:update-file-result
|
||||
(sm/define
|
||||
[:vector {:title "update-file-result"}
|
||||
[:map
|
||||
[:changes [:vector ::cpc/change]]
|
||||
[:file-id ::sm/uuid]
|
||||
[:id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:session-id ::sm/uuid]]]))
|
||||
(sm/def! ::change-with-metadata
|
||||
[:map {:title "ChangeWithMetadata"}
|
||||
[:changes ::changes]
|
||||
[:hint-origin {:optional true} :keyword]
|
||||
[:hint-events {:optional true} [:vector :string]]])
|
||||
|
||||
(sm/def! ::update-file-params
|
||||
[:map {:title "UpdateFileParams"}
|
||||
[:id ::sm/uuid]
|
||||
[:session-id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:features {:optional true
|
||||
:gen/max 3
|
||||
:gen/gen (smg/subseq files/supported-features)}
|
||||
::sm/set-of-strings]
|
||||
[:changes {:optional true} ::changes]
|
||||
[:changes-with-metadata {:optional true}
|
||||
[:vector ::change-with-metadata]]])
|
||||
|
||||
(sm/def! ::update-file-result
|
||||
[:vector {:title "UpdateFileResults"}
|
||||
[:map {:title "UpdateFileResult"}
|
||||
[:changes ::changes]
|
||||
[:file-id ::sm/uuid]
|
||||
[:id ::sm/uuid]
|
||||
[:revn {:min 0} :int]
|
||||
[:session-id ::sm/uuid]]])
|
||||
|
||||
;; --- HELPERS
|
||||
|
||||
@@ -74,7 +105,7 @@
|
||||
(def ^:private library-change-types
|
||||
#{:add-color :mod-color :del-color
|
||||
:add-media :mod-media :del-media
|
||||
:add-component :mod-component :del-component :restore-component
|
||||
:add-component :mod-component :del-component
|
||||
:add-typography :mod-typography :del-typography})
|
||||
|
||||
(def ^:private file-change-types
|
||||
@@ -106,25 +137,24 @@
|
||||
|
||||
(defn- wrap-with-pointer-map-context
|
||||
[f]
|
||||
(fn [cfg {:keys [id] :as file}]
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)
|
||||
pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
|
||||
cfeat/*wrap-with-pointer-map-fn* pmap/wrap]
|
||||
(fn [{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
|
||||
(binding [pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn id)
|
||||
ffeat/*wrap-with-pointer-map-fn* pmap/wrap]
|
||||
(let [result (f cfg file)]
|
||||
(feat.fdata/persist-pointers! cfg id)
|
||||
(files/persist-pointers! conn id)
|
||||
result))))
|
||||
|
||||
(defn- wrap-with-objects-map-context
|
||||
[f]
|
||||
(fn [cfg file]
|
||||
(binding [cfeat/*wrap-with-objects-map-fn* omap/wrap]
|
||||
(binding [ffeat/*wrap-with-objects-map-fn* omap/wrap]
|
||||
(f cfg file))))
|
||||
|
||||
(declare get-lagged-changes)
|
||||
(declare send-notifications!)
|
||||
(declare update-file)
|
||||
(declare update-file*)
|
||||
(declare update-file-data)
|
||||
(declare take-snapshot?)
|
||||
|
||||
;; If features are specified from params and the final feature
|
||||
@@ -132,94 +162,73 @@
|
||||
;; database.
|
||||
|
||||
(sv/defmethod ::update-file
|
||||
{::climit/id :update-file/by-profile
|
||||
::climit/key-fn ::rpc/profile-id
|
||||
{::climit/id :update-file-by-id
|
||||
::climit/key-fn :id
|
||||
::webhooks/event? true
|
||||
::webhooks/batch-timeout (dt/duration "2m")
|
||||
::webhooks/batch-key (webhooks/key-fn ::rpc/profile-id :id)
|
||||
|
||||
::sm/params schema:update-file
|
||||
::sm/result schema:update-file-result
|
||||
::sm/params ::update-file-params
|
||||
::sm/result ::update-file-result
|
||||
|
||||
::doc/module :files
|
||||
::doc/added "1.17"}
|
||||
[cfg {:keys [::rpc/profile-id id] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(files/check-edition-permissions! conn profile-id id)
|
||||
(db/xact-lock! conn id)
|
||||
|
||||
(let [file (get-file conn id)
|
||||
team (teams/get-team conn
|
||||
:profile-id profile-id
|
||||
:team-id (:team-id file))
|
||||
|
||||
features (-> (cfeat/get-team-enabled-features cf/flags team)
|
||||
(cfeat/check-client-features! (:features params))
|
||||
(cfeat/check-file-features! (:features file) (:features params)))
|
||||
|
||||
params (assoc params
|
||||
:profile-id profile-id
|
||||
:features features
|
||||
:team team
|
||||
:file file)
|
||||
|
||||
tpoint (dt/tpoint)]
|
||||
|
||||
;; When newly computed features does not match exactly with
|
||||
;; the features defined on team row, we update it.
|
||||
(when (not= features (:features team))
|
||||
(let [features (db/create-array conn "text" features)]
|
||||
(db/update! conn :team
|
||||
{:features features}
|
||||
{:id (:id team)})))
|
||||
|
||||
(binding [l/*context* (some-> (meta params)
|
||||
(get :app.http/request)
|
||||
(errors/request->context))]
|
||||
(-> (update-file cfg params)
|
||||
(rph/with-defer #(let [elapsed (tpoint)]
|
||||
(l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))))
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(files/check-edition-permissions! conn profile-id id)
|
||||
(db/xact-lock! conn id)
|
||||
(let [cfg (assoc cfg ::db/conn conn)
|
||||
params (assoc params :profile-id profile-id)
|
||||
tpoint (dt/tpoint)]
|
||||
(-> (update-file cfg params)
|
||||
(rph/with-defer #(let [elapsed (tpoint)]
|
||||
(l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))
|
||||
|
||||
(defn update-file
|
||||
[{:keys [::db/conn ::mtx/metrics] :as cfg}
|
||||
{:keys [id file features changes changes-with-metadata] :as params}]
|
||||
(let [features (-> features
|
||||
(set/difference cfeat/frontend-only-features)
|
||||
(set/union (:features file)))
|
||||
[{:keys [::db/conn ::mtx/metrics] :as cfg} {:keys [profile-id id changes changes-with-metadata] :as params}]
|
||||
(let [file (get-file conn id)
|
||||
features (->> (concat (:features file)
|
||||
(:features params))
|
||||
(into (files/get-default-features))
|
||||
(files/check-features-compatibility!))]
|
||||
|
||||
update-fn (cond-> update-file*
|
||||
(contains? features "fdata/pointer-map")
|
||||
(wrap-with-pointer-map-context)
|
||||
(files/check-edition-permissions! conn profile-id (:id file))
|
||||
|
||||
(contains? features "fdata/objects-map")
|
||||
(wrap-with-objects-map-context))
|
||||
(binding [ffeat/*current* features
|
||||
ffeat/*previous* (:features file)]
|
||||
|
||||
changes (if changes-with-metadata
|
||||
(->> changes-with-metadata (mapcat :changes) vec)
|
||||
(vec changes))]
|
||||
(let [update-fn (cond-> update-file*
|
||||
(contains? features "storage/pointer-map")
|
||||
(wrap-with-pointer-map-context)
|
||||
|
||||
(when (> (:revn params)
|
||||
(:revn file))
|
||||
(ex/raise :type :validation
|
||||
:code :revn-conflict
|
||||
:hint "The incoming revision number is greater that stored version."
|
||||
:context {:incoming-revn (:revn params)
|
||||
:stored-revn (:revn file)}))
|
||||
(contains? features "storage/objects-map")
|
||||
(wrap-with-objects-map-context))
|
||||
|
||||
(mtx/run! metrics {:id :update-file-changes :inc (count changes)})
|
||||
file (assoc file :features features)
|
||||
changes (if changes-with-metadata
|
||||
(->> changes-with-metadata (mapcat :changes) vec)
|
||||
(vec changes))
|
||||
|
||||
(when (not= features (:features file))
|
||||
(let [features (db/create-array conn "text" features)]
|
||||
(db/update! conn :file
|
||||
{:features features}
|
||||
{:id id})))
|
||||
params (-> params
|
||||
(assoc :file file)
|
||||
(assoc :changes changes)
|
||||
(assoc ::created-at (dt/now)))]
|
||||
|
||||
(binding [cfeat/*current* features
|
||||
cfeat/*previous* (:features file)]
|
||||
(let [file (assoc file :features features)
|
||||
params (-> params
|
||||
(assoc :file file)
|
||||
(assoc :changes changes)
|
||||
(assoc ::created-at (dt/now)))]
|
||||
(when (> (:revn params)
|
||||
(:revn file))
|
||||
(ex/raise :type :validation
|
||||
:code :revn-conflict
|
||||
:hint "The incoming revision number is greater that stored version."
|
||||
:context {:incoming-revn (:revn params)
|
||||
:stored-revn (:revn file)}))
|
||||
|
||||
(mtx/run! metrics {:id :update-file-changes :inc (count changes)})
|
||||
|
||||
(when (not= features (:features file))
|
||||
(let [features (db/create-array conn "text" features)]
|
||||
(db/update! conn :file
|
||||
{:features features}
|
||||
{:id id})))
|
||||
|
||||
(-> (update-fn cfg params)
|
||||
(vary-meta assoc ::audit/replace-props
|
||||
@@ -229,16 +238,33 @@
|
||||
:project-id (:project-id file)
|
||||
:team-id (:team-id file)}))))))
|
||||
|
||||
(defn- update-file-data
|
||||
[file changes]
|
||||
(-> file
|
||||
(update :revn inc)
|
||||
(update :data (fn [data]
|
||||
(cond-> data
|
||||
:always
|
||||
(-> (blob/decode)
|
||||
(assoc :id (:id file))
|
||||
(pmg/migrate-data))
|
||||
|
||||
(and (contains? ffeat/*current* "components/v2")
|
||||
(not (contains? ffeat/*previous* "components/v2")))
|
||||
(ctf/migrate-to-components-v2)
|
||||
|
||||
:always
|
||||
(-> (cp/process-changes changes)
|
||||
(blob/encode)))))))
|
||||
|
||||
|
||||
(defn- update-file*
|
||||
[{:keys [::db/conn ::wrk/executor] :as cfg}
|
||||
{:keys [profile-id file changes session-id ::created-at skip-validate] :as params}]
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file changes session-id ::created-at] :as params}]
|
||||
(let [;; Process the file data in the CLIMIT context; scheduling it
|
||||
;; to be executed on a separated executor for avoid to do the
|
||||
;; CPU intensive operation on vthread.
|
||||
|
||||
update-fdata-fn (partial update-file-data cfg file changes skip-validate)
|
||||
file (-> (climit/configure cfg :update-file/global)
|
||||
(climit/run! update-fdata-fn executor))]
|
||||
file (-> (climit/configure cfg :update-file)
|
||||
(climit/submit! (partial update-file-data file changes)))]
|
||||
|
||||
(db/insert! conn :file-change
|
||||
{:id (uuid/next)
|
||||
@@ -271,74 +297,6 @@
|
||||
;; Retrieve and return lagged data
|
||||
(get-lagged-changes conn params))))
|
||||
|
||||
(defn- soft-validate-file-schema!
|
||||
[file]
|
||||
(try
|
||||
(val/validate-file-schema! file)
|
||||
(catch Throwable cause
|
||||
(l/error :hint "file schema validation error" :cause cause))))
|
||||
|
||||
(defn- soft-validate-file!
|
||||
[file libs]
|
||||
(try
|
||||
(val/validate-file! file libs)
|
||||
(catch Throwable cause
|
||||
(l/error :hint "file validation error"
|
||||
:cause cause))))
|
||||
|
||||
(defn- update-file-data
|
||||
[{:keys [::db/conn] :as cfg} file changes skip-validate]
|
||||
(let [file (update file :data (fn [data]
|
||||
(-> data
|
||||
(blob/decode)
|
||||
(assoc :id (:id file))
|
||||
(fmg/migrate-data)
|
||||
(d/without-nils))))
|
||||
|
||||
;; WARNING: this ruins performance; maybe we need to find
|
||||
;; some other way to do general validation
|
||||
libs (when (and (or (contains? cf/flags :file-validation)
|
||||
(contains? cf/flags :soft-file-validation))
|
||||
(not skip-validate))
|
||||
(->> (files/get-file-libraries conn (:id file))
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)
|
||||
pmap/*tracked* nil]
|
||||
(-> (files/get-file cfg id :migrate? false)
|
||||
(feat.fdata/process-pointers deref) ; ensure all pointers resolved
|
||||
(fmg/migrate-file))))))
|
||||
(d/index-by :id)))
|
||||
|
||||
file (-> (files/check-version! file)
|
||||
(update :revn inc)
|
||||
(update :data cpc/process-changes changes))]
|
||||
|
||||
(when (contains? cf/flags :soft-file-validation)
|
||||
(soft-validate-file! file libs))
|
||||
|
||||
(when (contains? cf/flags :soft-file-schema-validation)
|
||||
(soft-validate-file-schema! file))
|
||||
|
||||
(when (and (contains? cf/flags :file-validation)
|
||||
(not skip-validate))
|
||||
(val/validate-file! file libs))
|
||||
|
||||
(when (and (contains? cf/flags :file-schema-validation)
|
||||
(not skip-validate))
|
||||
(val/validate-file-schema! file))
|
||||
|
||||
(cond-> file
|
||||
(and (contains? cfeat/*current* "fdata/objects-map")
|
||||
(not (contains? cfeat/*previous* "fdata/objects-map")))
|
||||
(feat.fdata/enable-objects-map)
|
||||
|
||||
(and (contains? cfeat/*current* "fdata/pointer-map")
|
||||
(not (contains? cfeat/*previous* "fdata/pointer-map")))
|
||||
(feat.fdata/enable-pointer-map)
|
||||
|
||||
:always
|
||||
(update :data blob/encode))))
|
||||
|
||||
(defn- take-snapshot?
|
||||
"Defines the rule when file `data` snapshot should be saved."
|
||||
[{:keys [revn modified-at] :as file}]
|
||||
@@ -366,7 +324,7 @@
|
||||
(vec)))
|
||||
|
||||
(defn- send-notifications!
|
||||
[cfg {:keys [file team changes session-id] :as params}]
|
||||
[{:keys [::db/conn] :as cfg} {:keys [file changes session-id] :as params}]
|
||||
(let [lchanges (filter library-change? changes)
|
||||
msgbus (::mbus/msgbus cfg)]
|
||||
|
||||
@@ -380,12 +338,14 @@
|
||||
:changes changes})
|
||||
|
||||
(when (and (:is-shared file) (seq lchanges))
|
||||
(mbus/pub! msgbus
|
||||
:topic (:id team)
|
||||
:message {:type :library-change
|
||||
:profile-id (:profile-id params)
|
||||
:file-id (:id file)
|
||||
:session-id session-id
|
||||
:revn (:revn file)
|
||||
:modified-at (dt/now)
|
||||
:changes lchanges}))))
|
||||
(let [team-id (or (:team-id file)
|
||||
(files/get-team-id conn (:project-id file)))]
|
||||
(mbus/pub! msgbus
|
||||
:topic team-id
|
||||
:message {:type :library-change
|
||||
:profile-id (:profile-id params)
|
||||
:file-id (:id file)
|
||||
:session-id session-id
|
||||
:revn (:revn file)
|
||||
:modified-at (dt/now)
|
||||
:changes lchanges})))))
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
[app.storage :as sto]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.spec.alpha :as s]))
|
||||
|
||||
(def valid-weight #{100 200 300 400 500 600 700 800 900 950})
|
||||
@@ -37,7 +36,6 @@
|
||||
(s/def ::id ::us/uuid)
|
||||
(s/def ::name ::us/not-empty-string)
|
||||
(s/def ::project-id ::us/uuid)
|
||||
(s/def ::share-id ::us/uuid)
|
||||
(s/def ::style valid-style)
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::weight valid-weight)
|
||||
@@ -49,8 +47,7 @@
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:opt-un [::team-id
|
||||
::file-id
|
||||
::project-id
|
||||
::share-id])
|
||||
::project-id])
|
||||
(fn [o]
|
||||
(or (contains? o :team-id)
|
||||
(contains? o :file-id)
|
||||
@@ -58,7 +55,7 @@
|
||||
|
||||
(sv/defmethod ::get-font-variants
|
||||
{::doc/added "1.18"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id file-id project-id share-id] :as params}]
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id file-id project-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(cond
|
||||
(uuid? team-id)
|
||||
@@ -77,9 +74,8 @@
|
||||
|
||||
(uuid? file-id)
|
||||
(let [file (db/get-by-id conn :file file-id {:columns [:id :project-id]})
|
||||
project (db/get-by-id conn :project (:project-id file) {:columns [:id :team-id]})
|
||||
perms (files/get-permissions conn profile-id file-id share-id)]
|
||||
(files/check-read-permissions! perms)
|
||||
project (db/get-by-id conn :project (:project-id file) {:columns [:id :team-id]})]
|
||||
(files/check-read-permissions! conn profile-id file-id)
|
||||
(db/query conn :team-font-variant
|
||||
{:team-id (:team-id project)
|
||||
:deleted-at nil})))))
|
||||
@@ -157,11 +153,11 @@
|
||||
:woff1-file-id (:id woff1)
|
||||
:woff2-file-id (:id woff2)
|
||||
:otf-file-id (:id otf)
|
||||
:ttf-file-id (:id ttf)}))]
|
||||
:ttf-file-id (:id ttf)}))
|
||||
]
|
||||
|
||||
(let [data (-> (climit/configure cfg :process-font/global)
|
||||
(climit/run! (partial generate-missing! data)
|
||||
(::wrk/executor cfg)))
|
||||
(let [data (-> (climit/configure cfg :process-font)
|
||||
(climit/submit! (partial generate-missing! data)))
|
||||
assets (persist-fonts-files! data)
|
||||
result (insert-font-variant! assets)]
|
||||
(vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))
|
||||
|
||||
@@ -38,8 +38,7 @@
|
||||
"Performs the authentication using LDAP backend. Only works if LDAP
|
||||
is properly configured and enabled with `login-with-ldap` flag."
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::doc/module :auth}
|
||||
::doc/added "1.15"}
|
||||
[{:keys [::main/props ::ldap/provider] :as cfg} params]
|
||||
(when-not provider
|
||||
(ex/raise :type :restriction
|
||||
@@ -78,13 +77,13 @@
|
||||
::audit/profile-id (:id profile)}))))))
|
||||
|
||||
(defn- login-or-register
|
||||
[cfg info]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(or (some->> (:email info)
|
||||
(profile/get-profile-by-email conn)
|
||||
(profile/decode-row))
|
||||
(->> (assoc info :is-active true :is-demo false)
|
||||
(auth/create-profile! conn)
|
||||
(auth/create-profile-rels! conn)
|
||||
(profile/strip-private-attrs))))))
|
||||
[{:keys [::db/pool] :as cfg} info]
|
||||
(db/with-atomic [conn pool]
|
||||
(or (some->> (:email info)
|
||||
(profile/get-profile-by-email conn)
|
||||
(profile/decode-row))
|
||||
(->> (assoc info :is-active true :is-demo false)
|
||||
(auth/create-profile! conn)
|
||||
(auth/create-profile-rels! conn)
|
||||
(profile/strip-private-attrs)))))
|
||||
|
||||
|
||||
@@ -9,50 +9,46 @@
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.migrations :as pmg]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.http.sse :as sse]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.binfile :as binfile]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.projects :as proj]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.setup :as-alias setup]
|
||||
[app.setup.templates :as tmpl]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.walk :as walk]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]))
|
||||
[clojure.spec.alpha :as s]
|
||||
[clojure.walk :as walk]))
|
||||
|
||||
;; --- COMMAND: Duplicate File
|
||||
|
||||
(declare duplicate-file)
|
||||
|
||||
(def ^:private
|
||||
schema:duplicate-file
|
||||
(sm/define
|
||||
[:map {:title "duplicate-file"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:name {:optional true} :string]]))
|
||||
(s/def ::id ::us/uuid)
|
||||
(s/def ::project-id ::us/uuid)
|
||||
(s/def ::file-id ::us/uuid)
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::name ::us/string)
|
||||
|
||||
(s/def ::duplicate-file
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::file-id]
|
||||
:opt-un [::name]))
|
||||
|
||||
(sv/defmethod ::duplicate-file
|
||||
"Duplicate a single file in the same team."
|
||||
{::doc/added "1.16"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:duplicate-file}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(db/tx-run! cfg duplicate-file (assoc params :profile-id profile-id)))
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(duplicate-file conn (assoc params :profile-id profile-id))))
|
||||
|
||||
(defn- remap-id
|
||||
[item index key]
|
||||
@@ -61,7 +57,7 @@
|
||||
(assoc key (get index (get item key) (get item key)))))
|
||||
|
||||
(defn- process-file
|
||||
[cfg index {:keys [id] :as file}]
|
||||
[conn {:keys [id] :as file} index]
|
||||
(letfn [(process-form [form]
|
||||
(cond-> form
|
||||
;; Relink library items
|
||||
@@ -104,45 +100,35 @@
|
||||
(dissoc k))
|
||||
res)))
|
||||
media
|
||||
media))
|
||||
media))]
|
||||
(-> file
|
||||
(update :id #(get index %))
|
||||
(update :data
|
||||
(fn [data]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)
|
||||
pmap/*tracked* (atom {})]
|
||||
(let [file-id (get index id)
|
||||
data (-> data
|
||||
(blob/decode)
|
||||
(assoc :id file-id)
|
||||
(pmg/migrate-data)
|
||||
(update :pages-index relink-shapes)
|
||||
(update :components relink-shapes)
|
||||
(update :media relink-media)
|
||||
(d/without-nils)
|
||||
(files/process-pointers pmap/clone)
|
||||
(blob/encode))]
|
||||
(files/persist-pointers! conn file-id)
|
||||
data)))))))
|
||||
|
||||
(process-file [{:keys [id] :as file}]
|
||||
(-> file
|
||||
(update :data assoc :id id)
|
||||
(update :data feat.fdata/process-pointers deref)
|
||||
(pmg/migrate-file)
|
||||
(update :data (fn [data]
|
||||
(-> data
|
||||
(update :pages-index relink-shapes)
|
||||
(update :components relink-shapes)
|
||||
(update :media relink-media)
|
||||
(d/without-nils))))))]
|
||||
|
||||
(let [new-id (get index id)
|
||||
file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)]
|
||||
(-> (assoc file :id new-id)
|
||||
(process-file)))
|
||||
|
||||
file (if (contains? (:features file) "fdata/objects-map")
|
||||
(feat.fdata/enable-objects-map file)
|
||||
file)
|
||||
|
||||
file (if (contains? (:features file) "fdata/pointer-map")
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)]
|
||||
(let [file (feat.fdata/enable-pointer-map file)]
|
||||
(feat.fdata/persist-pointers! cfg (:id file))
|
||||
file))
|
||||
file)]
|
||||
file)))
|
||||
|
||||
(def sql:get-used-libraries
|
||||
(def sql:retrieve-used-libraries
|
||||
"select flr.*
|
||||
from file_library_rel as flr
|
||||
inner join file as l on (flr.library_file_id = l.id)
|
||||
where flr.file_id = ?
|
||||
and l.deleted_at is null")
|
||||
|
||||
(def sql:get-used-media-objects
|
||||
(def sql:retrieve-used-media-objects
|
||||
"select fmo.*
|
||||
from file_media_object as fmo
|
||||
inner join storage_object as so on (fmo.media_id = so.id)
|
||||
@@ -150,9 +136,9 @@
|
||||
and so.deleted_at is null")
|
||||
|
||||
(defn duplicate-file*
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
|
||||
(let [flibs (or flibs (db/exec! conn [sql:get-used-libraries (:id file)]))
|
||||
fmeds (or fmeds (db/exec! conn [sql:get-used-media-objects (:id file)]))
|
||||
[conn {:keys [profile-id file index project-id name flibs fmeds]} {:keys [reset-shared-flag]}]
|
||||
(let [flibs (or flibs (db/exec! conn [sql:retrieve-used-libraries (:id file)]))
|
||||
fmeds (or fmeds (db/exec! conn [sql:retrieve-used-media-objects (:id file)]))
|
||||
|
||||
;; memo uniform creation/modification date
|
||||
now (dt/now)
|
||||
@@ -193,72 +179,65 @@
|
||||
(assoc :modified-at now)
|
||||
(assoc :ignore-sync-until ignore))
|
||||
|
||||
file (process-file cfg index file)]
|
||||
|
||||
(db/insert! conn :file
|
||||
(-> file
|
||||
(update :features #(db/create-array conn "text" %))
|
||||
(update :data blob/encode))
|
||||
{::db/return-keys? false})
|
||||
file (process-file conn file index)]
|
||||
|
||||
(db/insert! conn :file file)
|
||||
(db/insert! conn :file-profile-rel
|
||||
{:file-id (:id file)
|
||||
:profile-id profile-id
|
||||
:is-owner true
|
||||
:is-admin true
|
||||
:can-edit true}
|
||||
{::db/return-keys? false})
|
||||
:can-edit true})
|
||||
|
||||
(doseq [params flibs]
|
||||
(db/insert! conn :file-library-rel params ::db/return-keys? false))
|
||||
(db/insert! conn :file-library-rel params))
|
||||
|
||||
(doseq [params fmeds]
|
||||
(db/insert! conn :file-media-object params ::db/return-keys? false))
|
||||
(db/insert! conn :file-media-object params))
|
||||
|
||||
file))
|
||||
|
||||
(defn duplicate-file
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file-id] :as params}]
|
||||
(let [;; We don't touch the original file on duplication
|
||||
file (files/get-file cfg file-id :migrate? false)
|
||||
[conn {:keys [profile-id file-id] :as params}]
|
||||
(let [file (db/get-by-id conn :file file-id)
|
||||
index {file-id (uuid/next)}
|
||||
params (assoc params :index index :file file)]
|
||||
(proj/check-edition-permissions! conn profile-id (:project-id file))
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
|
||||
(duplicate-file* cfg params {:reset-shared-flag true})))
|
||||
(-> (duplicate-file* conn params {:reset-shared-flag true})
|
||||
(update :data blob/decode)
|
||||
(update :features db/decode-pgarray #{}))))
|
||||
|
||||
;; --- COMMAND: Duplicate Project
|
||||
|
||||
(declare duplicate-project)
|
||||
|
||||
(def ^:private
|
||||
schema:duplicate-project
|
||||
(sm/define
|
||||
[:map {:title "duplicate-project"}
|
||||
[:project-id ::sm/uuid]
|
||||
[:name {:optional true} :string]]))
|
||||
(s/def ::duplicate-project
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::project-id]
|
||||
:opt-un [::name]))
|
||||
|
||||
(sv/defmethod ::duplicate-project
|
||||
"Duplicate an entire project with all the files"
|
||||
{::doc/added "1.16"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:duplicate-project}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(db/tx-run! cfg duplicate-project (assoc params :profile-id profile-id)))
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} params]
|
||||
(db/with-atomic [conn pool]
|
||||
(duplicate-project conn (assoc params :profile-id (::rpc/profile-id params)))))
|
||||
|
||||
(defn duplicate-project
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id project-id name] :as params}]
|
||||
[conn {:keys [profile-id project-id name] :as params}]
|
||||
|
||||
;; Defer all constraints
|
||||
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
|
||||
|
||||
(let [project (-> (db/get-by-id conn :project project-id)
|
||||
(assoc :is-pinned false))
|
||||
|
||||
|
||||
files (db/query conn :file
|
||||
{:project-id (:id project)
|
||||
:deleted-at nil}
|
||||
{:columns [:id]})
|
||||
{:project-id (:id project)
|
||||
:deleted-at nil}
|
||||
{:columns [:id]})
|
||||
|
||||
project (cond-> project
|
||||
(string? name)
|
||||
@@ -272,8 +251,8 @@
|
||||
|
||||
;; create the duplicated project and assign the current profile as
|
||||
;; a project owner
|
||||
(teams/create-project conn project)
|
||||
(teams/create-project-role conn profile-id (:id project) :owner)
|
||||
(create-project conn project)
|
||||
(create-project-role conn profile-id (:id project) :owner)
|
||||
|
||||
;; duplicate all files
|
||||
(let [index (reduce #(assoc %1 (:id %2) (uuid/next)) {} files)
|
||||
@@ -282,18 +261,18 @@
|
||||
(assoc :project-id (:id project))
|
||||
(assoc :index index))]
|
||||
(doseq [{:keys [id]} files]
|
||||
(let [file (files/get-file cfg id :migrate? false)
|
||||
(let [file (db/get-by-id conn :file id)
|
||||
params (assoc params :file file)
|
||||
opts {:reset-shared-flag false}]
|
||||
(duplicate-file* cfg params opts))))
|
||||
(duplicate-file* conn params opts))))
|
||||
|
||||
;; return the created project
|
||||
project))
|
||||
|
||||
;; --- COMMAND: Move file
|
||||
|
||||
(def sql:get-files
|
||||
"select id, features, project_id from file where id = ANY(?)")
|
||||
(def sql:retrieve-files
|
||||
"select id, project_id from file where id = ANY(?)")
|
||||
|
||||
(def sql:move-files
|
||||
"update file set project_id = ? where id = ANY(?)")
|
||||
@@ -314,20 +293,14 @@
|
||||
and rel.library_file_id = br.library_file_id")
|
||||
|
||||
(defn move-files
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id ids project-id] :as params}]
|
||||
[conn {:keys [profile-id ids project-id] :as params}]
|
||||
|
||||
(let [fids (db/create-array conn "uuid" ids)
|
||||
files (->> (db/exec! conn [sql:get-files fids])
|
||||
(map files/decode-row))
|
||||
files (db/exec! conn [sql:retrieve-files fids])
|
||||
source (into #{} (map :project-id) files)
|
||||
pids (->> (conj source project-id)
|
||||
(db/create-array conn "uuid"))]
|
||||
|
||||
(when (contains? source project-id)
|
||||
(ex/raise :type :validation
|
||||
:code :cant-move-to-same-project
|
||||
:hint "Unable to move a file to the same project"))
|
||||
|
||||
;; Check if we have permissions on the destination project
|
||||
(proj/check-edition-permissions! conn profile-id project-id)
|
||||
|
||||
@@ -335,15 +308,10 @@
|
||||
(doseq [project-id source]
|
||||
(proj/check-edition-permissions! conn profile-id project-id))
|
||||
|
||||
;; Check the team compatibility
|
||||
(let [orig-team (teams/get-team conn :profile-id profile-id :project-id (first source))
|
||||
dest-team (teams/get-team conn :profile-id profile-id :project-id project-id)]
|
||||
(cfeat/check-teams-compatibility! orig-team dest-team)
|
||||
|
||||
;; Check if all pending to move files are compaib
|
||||
(let [features (cfeat/get-team-enabled-features cf/flags dest-team)]
|
||||
(doseq [file files]
|
||||
(cfeat/check-file-features! features (:features file)))))
|
||||
(when (contains? source project-id)
|
||||
(ex/raise :type :validation
|
||||
:code :cant-move-to-same-project
|
||||
:hint "Unable to move a file to the same project"))
|
||||
|
||||
;; move all files to the project
|
||||
(db/exec-one! conn [sql:move-files project-id fids])
|
||||
@@ -351,65 +319,38 @@
|
||||
;; delete possible broken relations on moved files
|
||||
(db/exec-one! conn [sql:delete-broken-relations pids])
|
||||
|
||||
;; Update the modification date of the all affected projects
|
||||
;; ensuring that the destination project is the most recent one.
|
||||
(doseq [project-id (into (list project-id) source)]
|
||||
|
||||
;; NOTE: as this is executed on virtual thread, sleeping does
|
||||
;; not causes major issues, and allows an easy way to set a
|
||||
;; trully different modification date to each file.
|
||||
(px/sleep 10)
|
||||
(db/update! conn :project
|
||||
{:modified-at (dt/now)}
|
||||
{:id project-id}))
|
||||
|
||||
nil))
|
||||
|
||||
(def ^:private
|
||||
schema:move-files
|
||||
(sm/define
|
||||
[:map {:title "move-files"}
|
||||
[:ids ::sm/set-of-uuid]
|
||||
[:project-id ::sm/uuid]]))
|
||||
(s/def ::ids (s/every ::us/uuid :kind set?))
|
||||
(s/def ::move-files
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::ids ::project-id]))
|
||||
|
||||
(sv/defmethod ::move-files
|
||||
"Move a set of files from one project to other."
|
||||
{::doc/added "1.16"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:move-files}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(db/tx-run! cfg #(move-files % (assoc params :profile-id profile-id))))
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(move-files conn (assoc params :profile-id profile-id))))
|
||||
|
||||
;; --- COMMAND: Move project
|
||||
|
||||
(defn move-project
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id team-id project-id] :as params}]
|
||||
[conn {:keys [profile-id team-id project-id] :as params}]
|
||||
(let [project (db/get-by-id conn :project project-id {:columns [:id :team-id]})
|
||||
pids (->> (db/query conn :project {:team-id (:team-id project)} {:columns [:id]})
|
||||
(map :id)
|
||||
(db/create-array conn "uuid"))]
|
||||
|
||||
(teams/check-edition-permissions! conn profile-id (:team-id project))
|
||||
(teams/check-edition-permissions! conn profile-id team-id)
|
||||
|
||||
(when (= team-id (:team-id project))
|
||||
(ex/raise :type :validation
|
||||
:code :cant-move-to-same-team
|
||||
:hint "Unable to move a project to same team"))
|
||||
|
||||
(teams/check-edition-permissions! conn profile-id (:team-id project))
|
||||
(teams/check-edition-permissions! conn profile-id team-id)
|
||||
|
||||
;; Check the teams compatibility
|
||||
(let [orig-team (teams/get-team conn :profile-id profile-id :team-id (:team-id project))
|
||||
dest-team (teams/get-team conn :profile-id profile-id :team-id team-id)]
|
||||
(cfeat/check-teams-compatibility! orig-team dest-team)
|
||||
|
||||
;; Check if all pending to move files are compaib
|
||||
(let [features (cfeat/get-team-enabled-features cf/flags dest-team)]
|
||||
(doseq [file (->> (db/query conn :file
|
||||
{:project-id project-id}
|
||||
{:columns [:features]})
|
||||
(map files/decode-row))]
|
||||
(cfeat/check-file-features! features (:features file)))))
|
||||
|
||||
;; move project to the destination team
|
||||
(db/update! conn :project
|
||||
{:team-id team-id}
|
||||
@@ -420,74 +361,61 @@
|
||||
|
||||
nil))
|
||||
|
||||
(def ^:private
|
||||
schema:move-project
|
||||
(sm/define
|
||||
[:map {:title "move-project"}
|
||||
[:team-id ::sm/uuid]
|
||||
[:project-id ::sm/uuid]]))
|
||||
|
||||
(s/def ::move-project
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::team-id ::project-id]))
|
||||
|
||||
(sv/defmethod ::move-project
|
||||
"Move projects between teams"
|
||||
"Move projects between teams."
|
||||
{::doc/added "1.16"
|
||||
::webhooks/event? true
|
||||
::sm/params schema:move-project}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(db/tx-run! cfg #(move-project % (assoc params :profile-id profile-id))))
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(move-project conn (assoc params :profile-id profile-id))))
|
||||
|
||||
;; --- COMMAND: Clone Template
|
||||
|
||||
(def ^:private
|
||||
schema:clone-template
|
||||
(sm/define
|
||||
[:map {:title "clone-template"}
|
||||
[:project-id ::sm/uuid]
|
||||
[:template-id ::sm/word-string]]))
|
||||
(declare clone-template)
|
||||
|
||||
(declare ^:private clone-template)
|
||||
(s/def ::template-id ::us/not-empty-string)
|
||||
(s/def ::clone-template
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::project-id ::template-id]))
|
||||
|
||||
(sv/defmethod ::clone-template
|
||||
"Clone into the specified project the template by its id."
|
||||
{::doc/added "1.16"
|
||||
::sse/stream? true
|
||||
::webhooks/event? true
|
||||
::sm/params schema:clone-template}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id template-id] :as params}]
|
||||
(let [project (db/get-by-id pool :project project-id {:columns [:id :team-id]})
|
||||
_ (teams/check-edition-permissions! pool profile-id (:team-id project))
|
||||
template (tmpl/get-template-stream cfg template-id)
|
||||
params (-> cfg
|
||||
(assoc ::binfile/input template)
|
||||
(assoc ::binfile/project-id (:id project))
|
||||
(assoc ::binfile/profile-id profile-id)
|
||||
(assoc ::binfile/ignore-index-errors? true)
|
||||
(assoc ::binfile/migrate? true))]
|
||||
::webhooks/event? true}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(-> (assoc cfg :conn conn)
|
||||
(clone-template (assoc params :profile-id profile-id)))))
|
||||
|
||||
(defn- clone-template
|
||||
[{:keys [conn templates] :as cfg} {:keys [profile-id template-id project-id]}]
|
||||
(let [template (d/seek #(= (:id %) template-id) templates)
|
||||
project (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
|
||||
|
||||
(teams/check-edition-permissions! conn profile-id (:team-id project))
|
||||
|
||||
(when-not template
|
||||
(ex/raise :type :not-found
|
||||
:code :template-not-found
|
||||
:hint "template not found"))
|
||||
|
||||
(sse/response #(clone-template params))))
|
||||
(-> cfg
|
||||
(assoc ::binfile/input (:path template))
|
||||
(assoc ::binfile/project-id (:id project))
|
||||
(assoc ::binfile/ignore-index-errors? true)
|
||||
(assoc ::binfile/migrate? true)
|
||||
(binfile/import!))))
|
||||
|
||||
(defn- clone-template
|
||||
[{:keys [::wrk/executor ::binfile/project-id] :as params}]
|
||||
(db/tx-run! params
|
||||
(fn [{:keys [::db/conn] :as params}]
|
||||
;; NOTE: the importation process performs some operations that
|
||||
;; are not very friendly with virtual threads, and for avoid
|
||||
;; unexpected blocking of other concurrent operations we
|
||||
;; dispatch that operation to a dedicated executor.
|
||||
(let [result (p/thread-call executor (partial binfile/import! params))]
|
||||
(db/update! conn :project
|
||||
{:modified-at (dt/now)}
|
||||
{:id project-id})
|
||||
|
||||
(deref result)))))
|
||||
;; --- COMMAND: Retrieve list of builtin templates
|
||||
|
||||
;; --- COMMAND: Get list of builtin templates
|
||||
(s/def ::retrieve-list-of-builtin-templates any?)
|
||||
|
||||
(sv/defmethod ::get-builtin-templates
|
||||
{::doc/added "1.19"}
|
||||
(sv/defmethod ::retrieve-list-of-builtin-templates
|
||||
[cfg _params]
|
||||
(mapv #(select-keys % [:id :name]) (::setup/templates cfg)))
|
||||
(mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.services :as sv]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.io :as io]))
|
||||
@@ -61,7 +60,7 @@
|
||||
(files/check-edition-permissions! pool profile-id file-id)
|
||||
(media/validate-media-type! content)
|
||||
(media/validate-media-size! content)
|
||||
(let [object (db/run! cfg #(create-file-media-object % params))
|
||||
(let [object (create-file-media-object cfg params)
|
||||
props {:name (:name params)
|
||||
:file-id file-id
|
||||
:is-local (:is-local params)
|
||||
@@ -143,17 +142,17 @@
|
||||
(assoc ::image (process-main-image info)))))
|
||||
|
||||
(defn create-file-media-object
|
||||
[{:keys [::sto/storage ::db/conn ::wrk/executor] :as cfg}
|
||||
[{:keys [::sto/storage ::db/pool] :as cfg}
|
||||
{:keys [id file-id is-local name content]}]
|
||||
|
||||
(let [result (-> (climit/configure cfg :process-image/global)
|
||||
(climit/run! (partial process-image content) executor))
|
||||
(let [result (-> (climit/configure cfg :process-image)
|
||||
(climit/submit! (partial process-image content)))
|
||||
|
||||
image (sto/put-object! storage (::image result))
|
||||
thumb (when-let [params (::thumb result)]
|
||||
(sto/put-object! storage params))]
|
||||
|
||||
(db/exec-one! conn [sql:create-file-media-object
|
||||
(db/exec-one! pool [sql:create-file-media-object
|
||||
(or id (uuid/next))
|
||||
file-id is-local name
|
||||
(:id image)
|
||||
@@ -172,14 +171,13 @@
|
||||
:opt-un [::id ::name]))
|
||||
|
||||
(sv/defmethod ::create-file-media-object-from-url
|
||||
{::doc/added "1.17"
|
||||
::doc/deprecated "1.19"}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id] :as params}]
|
||||
(let [cfg (update cfg ::sto/storage media/configure-assets-storage)]
|
||||
(files/check-edition-permissions! pool profile-id file-id)
|
||||
(db/run! cfg #(create-file-media-object-from-url % params))))
|
||||
(create-file-media-object-from-url cfg params)))
|
||||
|
||||
(defn download-image
|
||||
(defn- download-image
|
||||
[{:keys [::http/client]} uri]
|
||||
(letfn [(parse-and-validate [{:keys [headers] :as response}]
|
||||
(let [size (some-> (get headers "content-length") d/parse-integer)
|
||||
@@ -210,6 +208,7 @@
|
||||
{:method :get :uri uri}
|
||||
{:response-type :input-stream :sync? true})
|
||||
{:keys [size mtype]} (parse-and-validate response)
|
||||
|
||||
path (tmp/tempfile :prefix "penpot.media.download.")
|
||||
written (io/write-to-file! body path :size size)]
|
||||
|
||||
@@ -223,6 +222,7 @@
|
||||
:path path
|
||||
:mtype mtype})))
|
||||
|
||||
|
||||
(defn- create-file-media-object-from-url
|
||||
[cfg {:keys [url name] :as params}]
|
||||
(let [content (download-image cfg url)
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.uuid :as uuid]
|
||||
@@ -26,7 +27,7 @@
|
||||
[app.tokens :as tokens]
|
||||
[app.util.services :as sv]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as-alias wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[cuerdas.core :as str]))
|
||||
|
||||
(declare check-profile-existence!)
|
||||
@@ -37,30 +38,30 @@
|
||||
(declare strip-private-attrs)
|
||||
(declare verify-password)
|
||||
|
||||
(def ^:private
|
||||
schema:profile
|
||||
(sm/define
|
||||
[:map {:title "Profile"}
|
||||
[:id ::sm/uuid]
|
||||
[:fullname [::sm/word-string {:max 250}]]
|
||||
[:email ::sm/email]
|
||||
[:is-active {:optional true} :boolean]
|
||||
[:is-blocked {:optional true} :boolean]
|
||||
[:is-demo {:optional true} :boolean]
|
||||
[:is-muted {:optional true} :boolean]
|
||||
[:created-at {:optional true} ::sm/inst]
|
||||
[:modified-at {:optional true} ::sm/inst]
|
||||
[:default-project-id {:optional true} ::sm/uuid]
|
||||
[:default-team-id {:optional true} ::sm/uuid]
|
||||
[:props {:optional true}
|
||||
[:map-of {:title "ProfileProps"} :keyword :any]]]))
|
||||
(def schema:profile
|
||||
[:map {:title "Profile"}
|
||||
[:id ::sm/uuid]
|
||||
[:fullname :string]
|
||||
[:email ::sm/email]
|
||||
[:is-active {:optional true} :boolean]
|
||||
[:is-blocked {:optional true} :boolean]
|
||||
[:is-demo {:optional true} :boolean]
|
||||
[:is-muted {:optional true} :boolean]
|
||||
[:created-at {:optional true} ::sm/inst]
|
||||
[:modified-at {:optional true} ::sm/inst]
|
||||
[:default-project-id {:optional true} ::sm/uuid]
|
||||
[:default-team-id {:optional true} ::sm/uuid]
|
||||
[:props {:optional true}
|
||||
[:map-of {:title "ProfileProps"} :keyword :any]]])
|
||||
|
||||
(def profile?
|
||||
(sm/pred-fn schema:profile))
|
||||
|
||||
;; --- QUERY: Get profile (own)
|
||||
|
||||
(sv/defmethod ::get-profile
|
||||
{::rpc/auth false
|
||||
::doc/added "1.18"
|
||||
::sm/params [:map]
|
||||
::sm/result schema:profile}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id]}]
|
||||
;; We need to return the anonymous profile object in two cases, when
|
||||
@@ -81,20 +82,19 @@
|
||||
|
||||
;; --- MUTATION: Update Profile (own)
|
||||
|
||||
(def ^:private
|
||||
schema:update-profile
|
||||
(sm/define
|
||||
[:map {:title "update-profile"}
|
||||
[:fullname [::sm/word-string {:max 250}]]
|
||||
[:lang {:optional true} [:string {:max 5}]]
|
||||
[:theme {:optional true} [:string {:max 250}]]]))
|
||||
|
||||
(sv/defmethod ::update-profile
|
||||
{::doc/added "1.0"
|
||||
::sm/params schema:update-profile
|
||||
::sm/params [:map {:title "UpdateProfileParams"}
|
||||
[:fullname {:min 1} :string]
|
||||
[:lang {:optional true} :string]
|
||||
[:theme {:optional true} :string]]
|
||||
::sm/result schema:profile}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id fullname lang theme] :as params}]
|
||||
|
||||
(dm/assert!
|
||||
"expected valid profile data"
|
||||
(profile? params))
|
||||
|
||||
(db/with-atomic [conn pool]
|
||||
;; NOTE: we need to retrieve the profile independently if we use
|
||||
;; it or not for explicit locking and avoid concurrent updates of
|
||||
@@ -106,7 +106,8 @@
|
||||
profile (-> profile
|
||||
(assoc :fullname fullname)
|
||||
(assoc :lang lang)
|
||||
(assoc :theme theme))]
|
||||
(assoc :theme theme))
|
||||
]
|
||||
|
||||
(db/update! conn :profile
|
||||
{:fullname fullname
|
||||
@@ -127,17 +128,11 @@
|
||||
(declare update-profile-password!)
|
||||
(declare invalidate-profile-session!)
|
||||
|
||||
(def ^:private
|
||||
schema:update-profile-password
|
||||
(sm/define
|
||||
[:map {:title "update-profile-password"}
|
||||
[:password [::sm/word-string {:max 500}]]
|
||||
;; Social registered users don't have old-password
|
||||
[:old-password {:optional true} [:maybe [::sm/word-string {:max 500}]]]]))
|
||||
|
||||
(sv/defmethod ::update-profile-password
|
||||
{:doc/added "1.0"
|
||||
::sm/params schema:update-profile-password
|
||||
::sm/params [:map {:title "UpdateProfilePasswordParams"}
|
||||
[:password :string]
|
||||
[:old-password :string]]
|
||||
::sm/result :nil}
|
||||
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id password] :as params}]
|
||||
@@ -183,15 +178,10 @@
|
||||
(declare upload-photo)
|
||||
(declare update-profile-photo)
|
||||
|
||||
(def ^:private
|
||||
schema:update-profile-photo
|
||||
(sm/define
|
||||
[:map {:title "update-profile-photo"}
|
||||
[:file ::media/upload]]))
|
||||
|
||||
(sv/defmethod ::update-profile-photo
|
||||
{:doc/added "1.1"
|
||||
::sm/params schema:update-profile-photo
|
||||
::sm/params [:map {:title "UpdateProfilePhotoParams"}
|
||||
[:file ::media/upload]]
|
||||
::sm/result :nil}
|
||||
[cfg {:keys [::rpc/profile-id file] :as params}]
|
||||
;; Validate incoming mime type
|
||||
@@ -238,9 +228,9 @@
|
||||
:content-type (:mtype thumb)}))
|
||||
|
||||
(defn upload-photo
|
||||
[{:keys [::sto/storage ::wrk/executor] :as cfg} {:keys [file]}]
|
||||
(let [params (-> (climit/configure cfg :process-image/global)
|
||||
(climit/run! (partial generate-thumbnail! file) executor))]
|
||||
[{:keys [::sto/storage] :as cfg} {:keys [file]}]
|
||||
(let [params (-> (climit/configure cfg :process-image)
|
||||
(climit/submit! (partial generate-thumbnail! file)))]
|
||||
(sto/put-object! storage params)))
|
||||
|
||||
|
||||
@@ -249,15 +239,11 @@
|
||||
(declare ^:private request-email-change!)
|
||||
(declare ^:private change-email-immediately!)
|
||||
|
||||
(def ^:private
|
||||
schema:request-email-change
|
||||
(sm/define
|
||||
[:map {:title "request-email-change"}
|
||||
[:email ::sm/email]]))
|
||||
(s/def ::request-email-change
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::email]))
|
||||
|
||||
(sv/defmethod ::request-email-change
|
||||
{::doc/added "1.0"
|
||||
::sm/params schema:request-email-change}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id email] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [profile (db/get-by-id conn :profile profile-id)
|
||||
@@ -318,15 +304,12 @@
|
||||
|
||||
;; --- MUTATION: Update Profile Props
|
||||
|
||||
(def ^:private
|
||||
schema:update-profile-props
|
||||
(sm/define
|
||||
[:map {:title "update-profile-props"}
|
||||
[:props [:map-of :keyword :any]]]))
|
||||
(s/def ::props map?)
|
||||
(s/def ::update-profile-props
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::props]))
|
||||
|
||||
(sv/defmethod ::update-profile-props
|
||||
{::doc/added "1.0"
|
||||
::sm/params schema:update-profile-props}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id props]}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [profile (get-profile conn profile-id ::db/for-update? true)
|
||||
@@ -346,12 +329,15 @@
|
||||
|
||||
(filter-props props))))
|
||||
|
||||
|
||||
;; --- MUTATION: Delete Profile
|
||||
|
||||
(declare ^:private get-owned-teams-with-participants)
|
||||
|
||||
(s/def ::delete-profile
|
||||
(s/keys :req [::rpc/profile-id]))
|
||||
|
||||
(sv/defmethod ::delete-profile
|
||||
{::doc/added "1.0"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [teams (get-owned-teams-with-participants conn profile-id)
|
||||
@@ -438,15 +424,13 @@
|
||||
(defn derive-password
|
||||
[cfg password]
|
||||
(when password
|
||||
(-> (climit/configure cfg :derive-password/global)
|
||||
(climit/run! (partial auth/derive-password password)
|
||||
(::wrk/executor cfg)))))
|
||||
(-> (climit/configure cfg :derive-password)
|
||||
(climit/submit! (partial auth/derive-password password)))))
|
||||
|
||||
(defn verify-password
|
||||
[cfg password password-data]
|
||||
(-> (climit/configure cfg :derive-password/global)
|
||||
(climit/run! (partial auth/verify-password password password-data)
|
||||
(::wrk/executor cfg))))
|
||||
(-> (climit/configure cfg :derive-password)
|
||||
(climit/submit! (partial auth/verify-password password password-data))))
|
||||
|
||||
(defn decode-row
|
||||
[{:keys [props] :as row}]
|
||||
|
||||
@@ -50,11 +50,11 @@
|
||||
is-owner (boolean (some :is-owner rows))
|
||||
is-admin (boolean (some :is-admin rows))
|
||||
can-edit (boolean (some :can-edit rows))]
|
||||
(when (seq rows)
|
||||
{:is-owner is-owner
|
||||
:is-admin (or is-owner is-admin)
|
||||
:can-edit (or is-owner is-admin can-edit)
|
||||
:can-read true})))
|
||||
(when (seq rows)
|
||||
{:is-owner is-owner
|
||||
:is-admin (or is-owner is-admin)
|
||||
:can-edit (or is-owner is-admin can-edit)
|
||||
:can-read true})))
|
||||
|
||||
(def has-edit-permissions?
|
||||
(perms/make-edition-predicate-fn get-permissions))
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
[app.common.spec :as us]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.files :refer [resolve-public-uri]]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.util.services :as sv]
|
||||
[clojure.spec.alpha :as s]))
|
||||
@@ -38,15 +37,12 @@
|
||||
)
|
||||
select distinct
|
||||
f.id,
|
||||
f.revn,
|
||||
f.project_id,
|
||||
f.created_at,
|
||||
f.modified_at,
|
||||
f.name,
|
||||
f.is_shared,
|
||||
ft.media_id
|
||||
f.is_shared
|
||||
from file as f
|
||||
left join file_thumbnail as ft on (ft.file_id = f.id and ft.revn = f.revn)
|
||||
inner join projects as pr on (f.project_id = pr.id)
|
||||
where f.name ilike ('%' || ? || '%')
|
||||
and (f.deleted_at is null or f.deleted_at > now())
|
||||
@@ -54,16 +50,10 @@
|
||||
|
||||
(defn search-files
|
||||
[conn profile-id team-id search-term]
|
||||
(->> (db/exec! conn [sql:search-files
|
||||
profile-id team-id
|
||||
profile-id team-id
|
||||
search-term])
|
||||
(mapv (fn [row]
|
||||
(if-let [media-id (:media-id row)]
|
||||
(-> row
|
||||
(dissoc :media-id)
|
||||
(assoc :thumbnail-uri (resolve-public-uri media-id)))
|
||||
(dissoc row :media-id))))))
|
||||
(db/exec! conn [sql:search-files
|
||||
profile-id team-id
|
||||
profile-id team-id
|
||||
search-term]))
|
||||
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::search-files ::us/string)
|
||||
@@ -74,7 +64,6 @@
|
||||
:opt-un [::search-term]))
|
||||
|
||||
(sv/defmethod ::search-files
|
||||
{::doc/added "1.17"
|
||||
::doc/module :files}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id team-id search-term]}]
|
||||
(some->> search-term (search-files pool profile-id team-id)))
|
||||
|
||||
@@ -9,9 +9,7 @@
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
@@ -56,11 +54,11 @@
|
||||
is-owner (boolean (some :is-owner rows))
|
||||
is-admin (boolean (some :is-admin rows))
|
||||
can-edit (boolean (some :can-edit rows))]
|
||||
(when (seq rows)
|
||||
{:is-owner is-owner
|
||||
:is-admin (or is-owner is-admin)
|
||||
:can-edit (or is-owner is-admin can-edit)
|
||||
:can-read true})))
|
||||
(when (seq rows)
|
||||
{:is-owner is-owner
|
||||
:is-admin (or is-owner is-admin)
|
||||
:can-edit (or is-owner is-admin can-edit)
|
||||
:can-read true})))
|
||||
|
||||
(def has-admin-permissions?
|
||||
(perms/make-admin-predicate-fn get-permissions))
|
||||
@@ -80,26 +78,22 @@
|
||||
(def check-read-permissions!
|
||||
(perms/make-check-fn has-read-permissions?))
|
||||
|
||||
(defn decode-row
|
||||
[{:keys [features] :as row}]
|
||||
(cond-> row
|
||||
(some? features) (assoc :features (db/decode-pgarray features #{}))))
|
||||
|
||||
;; --- Query: Teams
|
||||
|
||||
(declare get-teams)
|
||||
(declare retrieve-teams)
|
||||
|
||||
(def ^:private schema:get-teams
|
||||
[:map {:title "get-teams"}])
|
||||
(def counter (volatile! 0))
|
||||
|
||||
(s/def ::get-teams
|
||||
(s/keys :req [::rpc/profile-id]))
|
||||
|
||||
(sv/defmethod ::get-teams
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-teams}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(get-teams conn profile-id)))
|
||||
(retrieve-teams conn profile-id)))
|
||||
|
||||
(def sql:get-teams-with-permissions
|
||||
(def sql:teams
|
||||
"select t.*,
|
||||
tp.is_owner,
|
||||
tp.is_admin,
|
||||
@@ -124,77 +118,37 @@
|
||||
(dissoc :is-owner :is-admin :can-edit)
|
||||
(assoc :permissions permissions))))
|
||||
|
||||
(defn get-teams
|
||||
(defn retrieve-teams
|
||||
[conn profile-id]
|
||||
(let [profile (profile/get-profile conn profile-id)]
|
||||
(->> (db/exec! conn [sql:get-teams-with-permissions (:default-team-id profile) profile-id])
|
||||
(map decode-row)
|
||||
(map process-permissions)
|
||||
(vec))))
|
||||
(->> (db/exec! conn [sql:teams (:default-team-id profile) profile-id])
|
||||
(mapv process-permissions))))
|
||||
|
||||
;; --- Query: Team (by ID)
|
||||
|
||||
(declare get-team)
|
||||
(declare retrieve-team)
|
||||
|
||||
(def ^:private schema:get-team
|
||||
[:and
|
||||
[:map {:title "get-team"}
|
||||
[:id {:optional true} ::sm/uuid]
|
||||
[:file-id {:optional true} ::sm/uuid]]
|
||||
|
||||
[:fn (fn [params]
|
||||
(or (contains? params :id)
|
||||
(contains? params :file-id)))]])
|
||||
(s/def ::get-team
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::id]))
|
||||
|
||||
(sv/defmethod ::get-team
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-team}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id id file-id]}]
|
||||
(get-team pool :profile-id profile-id :team-id id :file-id file-id))
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(retrieve-team conn profile-id id)))
|
||||
|
||||
(defn get-team
|
||||
[conn & {:keys [profile-id team-id project-id file-id] :as params}]
|
||||
|
||||
(dm/assert!
|
||||
"connection or pool is mandatory"
|
||||
(or (db/connection? conn)
|
||||
(db/pool? conn)))
|
||||
|
||||
(dm/assert!
|
||||
"profile-id is mandatory"
|
||||
(uuid? profile-id))
|
||||
|
||||
(let [{:keys [default-team-id] :as profile} (profile/get-profile conn profile-id)
|
||||
result (cond
|
||||
(some? team-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions
|
||||
") SELECT * FROM teams WHERE id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id team-id]))
|
||||
|
||||
(some? project-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" WHERE p.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id project-id]))
|
||||
|
||||
(some? file-id)
|
||||
(let [sql (str "WITH teams AS (" sql:get-teams-with-permissions ") "
|
||||
"SELECT t.* FROM teams AS t "
|
||||
" JOIN project AS p ON (p.team_id = t.id) "
|
||||
" JOIN file AS f ON (f.project_id = p.id) "
|
||||
" WHERE f.id=?")]
|
||||
(db/exec-one! conn [sql default-team-id profile-id file-id]))
|
||||
|
||||
:else
|
||||
(throw (IllegalArgumentException. "invalid arguments")))]
|
||||
(defn retrieve-team
|
||||
[conn profile-id team-id]
|
||||
(let [profile (profile/get-profile conn profile-id)
|
||||
sql (str "WITH teams AS (" sql:teams ") SELECT * FROM teams WHERE id=?")
|
||||
result (db/exec-one! conn [sql (:default-team-id profile) profile-id team-id])]
|
||||
|
||||
(when-not result
|
||||
(ex/raise :type :not-found
|
||||
:code :team-does-not-exist))
|
||||
(-> result
|
||||
(decode-row)
|
||||
(process-permissions))))
|
||||
|
||||
(process-permissions result)))
|
||||
|
||||
;; --- Query: Team Members
|
||||
|
||||
@@ -210,48 +164,44 @@
|
||||
join profile as p on (p.id = tp.profile_id)
|
||||
where tp.team_id = ?")
|
||||
|
||||
(defn get-team-members
|
||||
(defn retrieve-team-members
|
||||
[conn team-id]
|
||||
(db/exec! conn [sql:team-members team-id]))
|
||||
|
||||
(def ^:private schema:get-team-memebrs
|
||||
[:map {:title "get-team-members"}
|
||||
[:team-id ::sm/uuid]])
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::get-team-members
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::team-id]))
|
||||
|
||||
(sv/defmethod ::get-team-members
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-team-memebrs}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id team-id)
|
||||
(get-team-members conn team-id)))
|
||||
(retrieve-team-members conn team-id)))
|
||||
|
||||
|
||||
;; --- Query: Team Users
|
||||
|
||||
(declare get-users)
|
||||
(declare get-team-for-file)
|
||||
(declare retrieve-users)
|
||||
(declare retrieve-team-for-file)
|
||||
|
||||
(def ^:private schema:get-team-users
|
||||
[:and {:title "get-team-users"}
|
||||
[:map
|
||||
[:team-id {:optional true} ::sm/uuid]
|
||||
[:file-id {:optional true} ::sm/uuid]]
|
||||
[:fn #(or (contains? % :team-id)
|
||||
(contains? % :file-id))]])
|
||||
(s/def ::get-team-users
|
||||
(s/and (s/keys :req [::rpc/profile-id]
|
||||
:opt-un [::team-id ::file-id])
|
||||
#(or (:team-id %) (:file-id %))))
|
||||
|
||||
(sv/defmethod ::get-team-users
|
||||
"Get team users by team-id or by file-id"
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-team-users}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id file-id]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(if team-id
|
||||
(do
|
||||
(check-read-permissions! conn profile-id team-id)
|
||||
(get-users conn team-id))
|
||||
(let [{team-id :id} (get-team-for-file conn file-id)]
|
||||
(retrieve-users conn team-id))
|
||||
(let [{team-id :id} (retrieve-team-for-file conn file-id)]
|
||||
(check-read-permissions! conn profile-id team-id)
|
||||
(get-users conn team-id)))))
|
||||
(retrieve-users conn team-id)))))
|
||||
|
||||
;; This is a similar query to team members but can contain more data
|
||||
;; because some user can be explicitly added to project or file (not
|
||||
@@ -282,44 +232,44 @@
|
||||
join file as f on (p.id = f.project_id)
|
||||
where f.id = ?")
|
||||
|
||||
(defn get-users
|
||||
(defn retrieve-users
|
||||
[conn team-id]
|
||||
(db/exec! conn [sql:team-users team-id team-id team-id]))
|
||||
|
||||
(defn get-team-for-file
|
||||
(defn retrieve-team-for-file
|
||||
[conn file-id]
|
||||
(->> [sql:team-by-file file-id]
|
||||
(db/exec-one! conn)))
|
||||
|
||||
;; --- Query: Team Stats
|
||||
|
||||
(declare get-team-stats)
|
||||
(declare retrieve-team-stats)
|
||||
|
||||
(def ^:private schema:get-team-stats
|
||||
[:map {:title "get-team-stats"}
|
||||
[:team-id ::sm/uuid]])
|
||||
(s/def ::get-team-stats
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::team-id]))
|
||||
|
||||
(sv/defmethod ::get-team-stats
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-team-stats}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id team-id)
|
||||
(get-team-stats conn team-id)))
|
||||
(retrieve-team-stats conn team-id)))
|
||||
|
||||
(def sql:team-stats
|
||||
"select (select count(*) from project where team_id = ?) as projects,
|
||||
(select count(*) from file as f join project as p on (p.id = f.project_id) where p.team_id = ?) as files")
|
||||
|
||||
(defn get-team-stats
|
||||
(defn retrieve-team-stats
|
||||
[conn team-id]
|
||||
(db/exec-one! conn [sql:team-stats team-id team-id]))
|
||||
|
||||
|
||||
;; --- Query: Team invitations
|
||||
|
||||
(def ^:private schema:get-team-invitations
|
||||
[:map {:title "get-team-invitations"}
|
||||
[:team-id ::sm/uuid]])
|
||||
(s/def ::get-team-invitations
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::team-id]))
|
||||
|
||||
(def sql:team-invitations
|
||||
"select email_to as email, role, (valid_until < now()) as expired
|
||||
@@ -331,8 +281,7 @@
|
||||
(mapv #(update % :role keyword))))
|
||||
|
||||
(sv/defmethod ::get-team-invitations
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:get-team-invitations}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id]}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(check-read-permissions! conn profile-id team-id)
|
||||
@@ -347,32 +296,25 @@
|
||||
(declare ^:private create-team-role)
|
||||
(declare ^:private create-team-default-project)
|
||||
|
||||
(def ^:private schema:create-team
|
||||
[:map {:title "create-team"}
|
||||
[:name :string]
|
||||
[:features {:optional true} ::cfeat/features]
|
||||
[:id {:optional true} ::sm/uuid]])
|
||||
(s/def ::create-team
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::name]
|
||||
:opt-un [::id]))
|
||||
|
||||
(sv/defmethod ::create-team
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:create-team}
|
||||
[cfg {:keys [::rpc/profile-id] :as params}]
|
||||
(db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
(quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
|
||||
::quotes/profile-id profile-id})
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(quotes/check-quote! conn {::quotes/id ::quotes/teams-per-profile
|
||||
::quotes/profile-id profile-id})
|
||||
|
||||
(let [features (-> (cfeat/get-enabled-features cf/flags)
|
||||
(cfeat/check-client-features! (:features params)))]
|
||||
(create-team cfg (assoc params
|
||||
:profile-id profile-id
|
||||
:features features))))))
|
||||
(create-team conn (assoc params :profile-id profile-id))))
|
||||
|
||||
(defn create-team
|
||||
"This is a complete team creation process, it creates the team
|
||||
object and all related objects (default role and default project)."
|
||||
[cfg-or-conn params]
|
||||
(let [conn (db/get-connection cfg-or-conn)
|
||||
team (create-team* conn params)
|
||||
[conn params]
|
||||
(let [team (create-team* conn params)
|
||||
params (assoc params
|
||||
:team-id (:id team)
|
||||
:role :owner)
|
||||
@@ -381,16 +323,13 @@
|
||||
(assoc team :default-project-id (:id project))))
|
||||
|
||||
(defn- create-team*
|
||||
[conn {:keys [id name is-default features] :as params}]
|
||||
[conn {:keys [id name is-default] :as params}]
|
||||
(let [id (or id (uuid/next))
|
||||
is-default (if (boolean? is-default) is-default false)
|
||||
features (db/create-array conn "text" features)
|
||||
team (db/insert! conn :team
|
||||
{:id id
|
||||
:name name
|
||||
:features features
|
||||
:is-default is-default})]
|
||||
(decode-row team)))
|
||||
is-default (if (boolean? is-default) is-default false)]
|
||||
(db/insert! conn :team
|
||||
{:id id
|
||||
:name name
|
||||
:is-default is-default})))
|
||||
|
||||
(defn- create-team-role
|
||||
[conn {:keys [profile-id team-id role] :as params}]
|
||||
@@ -456,7 +395,7 @@
|
||||
(defn leave-team
|
||||
[conn {:keys [profile-id id reassign-to]}]
|
||||
(let [perms (get-permissions conn profile-id id)
|
||||
members (get-team-members conn id)]
|
||||
members (retrieve-team-members conn id)]
|
||||
|
||||
(cond
|
||||
;; we can only proceed if there are more members in the team
|
||||
@@ -540,15 +479,10 @@
|
||||
|
||||
(s/def ::team-id ::us/uuid)
|
||||
(s/def ::member-id ::us/uuid)
|
||||
(s/def ::role #{:owner :admin :editor})
|
||||
|
||||
;; Temporarily disabled viewer role
|
||||
;; https://tree.taiga.io/project/penpot/issue/1083
|
||||
(def valid-roles
|
||||
#{:owner :admin :editor #_:viewer})
|
||||
|
||||
(def schema:role
|
||||
[::sm/one-of valid-roles])
|
||||
;; (s/def ::role #{:owner :admin :editor :viewer})
|
||||
(s/def ::role #{:owner :admin :editor})
|
||||
|
||||
(defn role->params
|
||||
[role]
|
||||
@@ -565,7 +499,7 @@
|
||||
;; convenience, if this becomes a bottleneck or problematic,
|
||||
;; we will change it to more efficient fetch mechanisms.
|
||||
(let [perms (get-permissions conn profile-id team-id)
|
||||
members (get-team-members conn team-id)
|
||||
members (retrieve-team-members conn team-id)
|
||||
member (d/seek #(= member-id (:id %)) members)
|
||||
|
||||
is-owner? (:is-owner perms)
|
||||
@@ -661,7 +595,7 @@
|
||||
|
||||
(defn update-team-photo
|
||||
[{:keys [::db/pool ::sto/storage] :as cfg} {:keys [profile-id team-id] :as params}]
|
||||
(let [team (get-team pool :profile-id profile-id :team-id team-id)
|
||||
(let [team (retrieve-team pool profile-id team-id)
|
||||
photo (profile/upload-photo cfg params)]
|
||||
|
||||
(db/with-atomic [conn pool]
|
||||
@@ -673,8 +607,8 @@
|
||||
|
||||
;; Save new photo
|
||||
(db/update! pool :team
|
||||
{:photo-id (:id photo)}
|
||||
{:id team-id})
|
||||
{:photo-id (:id photo)}
|
||||
{:id team-id})
|
||||
|
||||
(assoc team :photo-id (:id photo)))))
|
||||
|
||||
@@ -735,8 +669,7 @@
|
||||
(role->params role))]
|
||||
|
||||
;; Insert the invited member to the team
|
||||
(db/insert! conn :team-profile-rel params
|
||||
{::db/on-conflict-do-nothing? true})
|
||||
(db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
|
||||
|
||||
;; If profile is not yet verified, mark it as verified because
|
||||
;; accepting an invitation link serves as verification.
|
||||
@@ -786,22 +719,29 @@
|
||||
|
||||
itoken))))
|
||||
|
||||
(def ^:private schema:create-team-invitations
|
||||
[:map {:title "create-team-invitations"}
|
||||
[:team-id ::sm/uuid]
|
||||
[:role [::sm/one-of #{:owner :admin :editor}]]
|
||||
[:emails ::sm/set-of-emails]])
|
||||
(s/def ::email ::us/email)
|
||||
(s/def ::emails ::us/set-of-valid-emails)
|
||||
(s/def ::create-team-invitations
|
||||
(s/keys :req [::rpc/profile-id]
|
||||
:req-un [::team-id ::role]
|
||||
:opt-un [::email ::emails]))
|
||||
|
||||
(sv/defmethod ::create-team-invitations
|
||||
"A rpc call that allow to send a single or multiple invitations to
|
||||
join the team."
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:create-team-invitations}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id emails role] :as params}]
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id team-id email emails role] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [perms (get-permissions conn profile-id team-id)
|
||||
profile (db/get-by-id conn :profile profile-id)
|
||||
team (db/get-by-id conn :team team-id)]
|
||||
team (db/get-by-id conn :team team-id)
|
||||
|
||||
;; Members emails. We don't re-send inviation to already existing members
|
||||
member? (into #{}
|
||||
(map :email)
|
||||
(db/exec! conn [sql:team-members team-id]))
|
||||
|
||||
emails (cond-> (or emails #{}) (string? email) (conj email))]
|
||||
|
||||
(run! (partial quotes/check-quote! conn)
|
||||
(list {::quotes/id ::quotes/invitations-per-team
|
||||
@@ -824,13 +764,9 @@
|
||||
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
|
||||
|
||||
(let [cfg (assoc cfg ::db/conn conn)
|
||||
members (->> (db/exec! conn [sql:team-members team-id])
|
||||
(into #{} (map :email)))
|
||||
|
||||
invitations (into #{}
|
||||
invitations (into []
|
||||
(comp
|
||||
;; We don't re-send inviation to already existing members
|
||||
(remove (partial contains? members))
|
||||
(remove member?)
|
||||
(map (fn [email]
|
||||
{:email (str/lower email)
|
||||
:team team
|
||||
@@ -838,8 +774,7 @@
|
||||
:role role}))
|
||||
(keep (partial create-invitation cfg)))
|
||||
emails)]
|
||||
(with-meta {:total (count invitations)
|
||||
:invitations invitations}
|
||||
(with-meta invitations
|
||||
{::audit/props {:invitations (count invitations)}})))))
|
||||
|
||||
|
||||
@@ -850,24 +785,14 @@
|
||||
(s/merge ::create-team
|
||||
(s/keys :req-un [::emails ::role])))
|
||||
|
||||
|
||||
(def ^:private schema:create-team-with-invitations
|
||||
[:map {:title "create-team-with-invitations"}
|
||||
[:name :string]
|
||||
[:features {:optional true} ::cfeat/features]
|
||||
[:id {:optional true} ::sm/uuid]
|
||||
[:emails ::sm/set-of-emails]
|
||||
[:role schema:role]])
|
||||
|
||||
(sv/defmethod ::create-team-with-invitations
|
||||
{::doc/added "1.17"
|
||||
::sm/params schema:create-team-with-invitations}
|
||||
{::doc/added "1.17"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id emails role] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [params (assoc params :profile-id profile-id)
|
||||
cfg (assoc cfg ::db/conn conn)
|
||||
team (create-team cfg params)
|
||||
profile (db/get-by-id conn :profile profile-id)]
|
||||
team (create-team conn params)
|
||||
profile (db/get-by-id conn :profile profile-id)
|
||||
cfg (assoc cfg ::db/conn conn)]
|
||||
|
||||
;; Create invitations for all provided emails.
|
||||
(->> emails
|
||||
|
||||
@@ -34,8 +34,7 @@
|
||||
|
||||
(sv/defmethod ::verify-token
|
||||
{::rpc/auth false
|
||||
::doc/added "1.15"
|
||||
::doc/module :auth}
|
||||
::doc/added "1.15"}
|
||||
[{:keys [::db/pool] :as cfg} {:keys [token] :as params}]
|
||||
(db/with-atomic [conn pool]
|
||||
(let [claims (tokens/verify (::main/props cfg) {:token token})
|
||||
@@ -105,7 +104,7 @@
|
||||
::quotes/team-id team-id})
|
||||
|
||||
;; Insert the invited member to the team
|
||||
(db/insert! conn :team-profile-rel params {::db/on-conflict-do-nothing? true})
|
||||
(db/insert! conn :team-profile-rel params {:on-conflict-do-nothing true})
|
||||
|
||||
;; If profile is not yet verified, mark it as verified because
|
||||
;; accepting an invitation link serves as verification.
|
||||
|
||||
@@ -6,49 +6,27 @@
|
||||
|
||||
(ns app.rpc.commands.viewer
|
||||
(:require
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.schema :as sm]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.commands.comments :as comments]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.cond :as-alias cond]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
[app.util.services :as sv]))
|
||||
|
||||
;; --- QUERY: View Only Bundle
|
||||
|
||||
(defn- remove-not-allowed-pages
|
||||
[data allowed]
|
||||
(-> data
|
||||
(update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
|
||||
(update :pages-index select-keys allowed)))
|
||||
|
||||
(defn- get-view-only-bundle
|
||||
[{:keys [::db/conn] :as cfg} {:keys [profile-id file-id ::perms] :as params}]
|
||||
(let [file (files/get-file cfg file-id)
|
||||
|
||||
project (db/get conn :project
|
||||
{:id (:project-id file)}
|
||||
{:columns [:id :name :team-id]})
|
||||
|
||||
team (-> (db/get conn :team {:id (:team-id project)})
|
||||
(teams/decode-row))
|
||||
|
||||
_ (-> (cfeat/get-team-enabled-features cf/flags team)
|
||||
(cfeat/check-client-features! (:features params))
|
||||
(cfeat/check-file-features! (:features file)))
|
||||
|
||||
file (cond-> file
|
||||
(= :share-link (:type perms))
|
||||
(update :data remove-not-allowed-pages (:pages perms))
|
||||
|
||||
:always
|
||||
(update :data select-keys [:id :options :pages :pages-index :components]))
|
||||
(defn- get-project
|
||||
[conn id]
|
||||
(db/get-by-id conn :project id {:columns [:id :name :team-id]}))
|
||||
|
||||
(defn- get-bundle
|
||||
[conn file-id profile-id features]
|
||||
(let [file (files/get-file conn file-id features)
|
||||
project (get-project conn (:project-id file))
|
||||
libs (files/get-file-libraries conn file-id)
|
||||
users (comments/get-file-comments-users conn file-id profile-id)
|
||||
links (->> (db/query conn :share-link {:file-id file-id})
|
||||
@@ -63,43 +41,57 @@
|
||||
(dissoc :flags)))))
|
||||
|
||||
fonts (db/query conn :team-font-variant
|
||||
{:team-id (:id team)
|
||||
{:team-id (:team-id project)
|
||||
:deleted-at nil})]
|
||||
|
||||
{:users users
|
||||
{:file file
|
||||
:users users
|
||||
:fonts fonts
|
||||
:project project
|
||||
:share-links links
|
||||
:libraries libs
|
||||
:file file
|
||||
:team team
|
||||
:permissions perms}))
|
||||
:libraries libs}))
|
||||
|
||||
(def schema:get-view-only-bundle
|
||||
(defn- remove-not-allowed-pages
|
||||
[data allowed]
|
||||
(-> data
|
||||
(update :pages (fn [pages] (filterv #(contains? allowed %) pages)))
|
||||
(update :pages-index select-keys allowed)))
|
||||
|
||||
(defn get-view-only-bundle
|
||||
[conn {:keys [profile-id file-id share-id features] :as params}]
|
||||
(let [perms (files/get-permissions conn profile-id file-id share-id)
|
||||
bundle (-> (get-bundle conn file-id profile-id features)
|
||||
(assoc :permissions perms))]
|
||||
|
||||
;; When we have neither profile nor share, we just return a not
|
||||
;; found response to the user.
|
||||
(when-not perms
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "object not found"))
|
||||
|
||||
(update bundle :file
|
||||
(fn [file]
|
||||
(cond-> file
|
||||
(= :share-link (:type perms))
|
||||
(update :data remove-not-allowed-pages (:pages perms))
|
||||
|
||||
:always
|
||||
(update :data select-keys [:id :options :pages :pages-index :components]))))))
|
||||
|
||||
(sm/def! ::get-view-only-bundle
|
||||
[:map {:title "get-view-only-bundle"}
|
||||
[:file-id ::sm/uuid]
|
||||
[:share-id {:optional true} ::sm/uuid]
|
||||
[:features {:optional true} ::cfeat/features]])
|
||||
[:features {:optional true} ::files/features]])
|
||||
|
||||
(sv/defmethod ::get-view-only-bundle
|
||||
{::rpc/auth false
|
||||
::cond/get-object #(files/get-minimal-file %1 (:file-id %2))
|
||||
::cond/key-fn files/get-file-etag
|
||||
::cond/reuse-key? true
|
||||
::doc/added "1.17"
|
||||
::sm/params schema:get-view-only-bundle}
|
||||
[system {:keys [::rpc/profile-id file-id share-id] :as params}]
|
||||
(db/run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(let [perms (files/get-permissions conn profile-id file-id share-id)
|
||||
params (-> params
|
||||
(assoc ::perms perms)
|
||||
(assoc :profile-id profile-id))]
|
||||
|
||||
;; When we have neither profile nor share, we just return a not
|
||||
;; found response to the user.
|
||||
(when-not perms
|
||||
(ex/raise :type :not-found
|
||||
:code :object-not-found
|
||||
:hint "object not found"))
|
||||
|
||||
(get-view-only-bundle system params)))))
|
||||
|
||||
|
||||
::sm/params ::get-view-only-bundle}
|
||||
[{:keys [::db/pool]} {:keys [::rpc/profile-id] :as params}]
|
||||
(dm/with-open [conn (db/open pool)]
|
||||
(get-view-only-bundle conn (assoc params :profile-id profile-id))))
|
||||
|
||||
@@ -27,25 +27,16 @@
|
||||
[app.common.logging :as l]
|
||||
[app.rpc.helpers :as rph]
|
||||
[app.util.services :as-alias sv]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.hash :as bh]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
(def
|
||||
^{:dynamic true
|
||||
:doc "Runtime flag for enable/disable conditional processing of RPC methods."}
|
||||
*enabled* false)
|
||||
|
||||
(defn- encode
|
||||
[s]
|
||||
(-> s
|
||||
bh/blake2b-256
|
||||
bc/bytes->b64u
|
||||
bc/bytes->str))
|
||||
|
||||
(defn- fmt-key
|
||||
[s]
|
||||
(str "W/\"" (encode s) "\""))
|
||||
(str "W/\"" s "\""))
|
||||
|
||||
(defn wrap
|
||||
[_ f {:keys [::get-object ::key-fn ::reuse-key?] :as mdata}]
|
||||
@@ -55,9 +46,10 @@
|
||||
(fn [cfg {:keys [::key] :as params}]
|
||||
(if *enabled*
|
||||
(let [key' (when (or key reuse-key?)
|
||||
(some->> (get-object cfg params) (key-fn params) (fmt-key)))]
|
||||
(if (and (some? key) (= key key'))
|
||||
(fn [_] {::rres/status 304})
|
||||
(some-> (get-object cfg params) key-fn fmt-key))]
|
||||
(if (and (some? key)
|
||||
(= key key'))
|
||||
(fn [_] {::yrs/status 304})
|
||||
(let [result (f cfg params)
|
||||
etag (or (and reuse-key? key')
|
||||
(some-> result meta ::key fmt-key)
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
[app.common.schema.openapi :as oapi]
|
||||
[app.common.schema.registry :as sr]
|
||||
[app.config :as cf]
|
||||
[app.http.sse :as-alias sse]
|
||||
[app.loggers.webhooks :as-alias webhooks]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.util.json :as json]
|
||||
@@ -28,7 +27,7 @@
|
||||
[integrant.core :as ig]
|
||||
[malli.transform :as mt]
|
||||
[pretty-spec.core :as ps]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.response :as yrs]))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; DOC (human readable)
|
||||
@@ -55,15 +54,14 @@
|
||||
{:name (::sv/name mdata)
|
||||
:module (or (some-> (::module mdata) d/name)
|
||||
(-> (:ns mdata) (str/split ".") last))
|
||||
:auth (::rpc/auth mdata true)
|
||||
:sse (::sse/stream? mdata false)
|
||||
:auth (:auth mdata true)
|
||||
:webhook (::webhooks/event? mdata false)
|
||||
:docs (::sv/docstring mdata)
|
||||
:deprecated (::deprecated mdata)
|
||||
:added (::added mdata)
|
||||
:changes (some->> (::changes mdata) (partition-all 2) (map vec))
|
||||
:spec (fmt-spec mdata)
|
||||
:entrypoint (str (cf/get :public-uri) "/api/rpc/command/" (::sv/name mdata))
|
||||
:entrypoint (str (cf/get :public-uri) "/api/rpc/commands/" (::sv/name mdata))
|
||||
|
||||
:params-schema-js (fmt-schema :js mdata ::sm/params)
|
||||
:result-schema-js (fmt-schema :js mdata ::sm/result)
|
||||
@@ -77,7 +75,6 @@
|
||||
(->> methods
|
||||
(map val)
|
||||
(map first)
|
||||
(remove ::skip)
|
||||
(map get-context)
|
||||
(sort-by (juxt :module :name)))}))
|
||||
|
||||
@@ -88,11 +85,11 @@
|
||||
(let [params (:query-params request)
|
||||
pstyle (:type params "js")
|
||||
context (assoc context :param-style pstyle)]
|
||||
{::rres/status 200
|
||||
::rres/body (-> (io/resource "app/templates/api-doc.tmpl")
|
||||
(tmpl/render context))}))
|
||||
{::yrs/status 200
|
||||
::yrs/body (-> (io/resource "app/templates/api-doc.tmpl")
|
||||
(tmpl/render context))}))
|
||||
(fn [_]
|
||||
{::rres/status 404})))
|
||||
{::yrs/status 404})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; OPENAPI / SWAGGER (v3.1)
|
||||
@@ -143,7 +140,8 @@
|
||||
|
||||
{:name (-> mdata ::sv/name d/name)
|
||||
:module (-> (:ns mdata) (str/split ".") last)
|
||||
:repr {:post rpost}}))]
|
||||
:repr {:post rpost}}))
|
||||
]
|
||||
|
||||
(let [definitions (atom {})
|
||||
options {:registry sr/default-registry
|
||||
@@ -157,29 +155,29 @@
|
||||
(map (partial gen-method-doc options))
|
||||
(sort-by (juxt :module :name))
|
||||
(map (fn [doc]
|
||||
[(str/ffmt "/command/%" (:name doc)) (:repr doc)]))
|
||||
[(str/ffmt "/commands/%" (:name doc)) (:repr doc)]))
|
||||
(into {})))]
|
||||
{:openapi "3.0.0"
|
||||
:info {:version (:main cf/version)}
|
||||
:servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
|
||||
{:openapi "3.0.0"
|
||||
:info {:version (:main cf/version)}
|
||||
:servers [{:url (str/ffmt "%/api/rpc" (cf/get :public-uri))
|
||||
;; :description "penpot backend"
|
||||
}]
|
||||
:security
|
||||
{:api_key []}
|
||||
}]
|
||||
:security
|
||||
{:api_key []}
|
||||
|
||||
:paths paths
|
||||
:components {:schemas @definitions}})))
|
||||
:paths paths
|
||||
:components {:schemas @definitions}})))
|
||||
|
||||
(defn openapi-json-handler
|
||||
[context]
|
||||
(if (contains? cf/flags :backend-openapi-doc)
|
||||
(fn [_]
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::rres/body (json/encode context)})
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "application/json; charset=utf-8"}
|
||||
::yrs/body (json/encode context)})
|
||||
|
||||
(fn [_]
|
||||
{::rres/status 404})))
|
||||
{::yrs/status 404})))
|
||||
|
||||
(defn openapi-handler
|
||||
[]
|
||||
@@ -190,12 +188,12 @@
|
||||
context {:public-uri (cf/get :public-uri)
|
||||
:swagger-js swagger-js
|
||||
:swagger-css swagger-cs}]
|
||||
{::rres/status 200
|
||||
::rres/headers {"content-type" "text/html"}
|
||||
::rres/body (-> (io/resource "app/templates/openapi.tmpl")
|
||||
(tmpl/render context))}))
|
||||
{::yrs/status 200
|
||||
::yrs/headers {"content-type" "text/html"}
|
||||
::yrs/body (-> (io/resource "app/templates/openapi.tmpl")
|
||||
(tmpl/render context))}))
|
||||
(fn [_]
|
||||
{::rres/status 404})))
|
||||
{::yrs/status 404})))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; MODULE INIT
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
[app.common.data.macros :as dm]
|
||||
[app.http :as-alias http]
|
||||
[app.rpc :as-alias rpc]
|
||||
[ring.response :as-alias rres]))
|
||||
[yetti.response :as-alias yrs]))
|
||||
|
||||
;; A utilty wrapper object for wrap service responses that does not
|
||||
;; implements the IObj interface that make possible attach metadata to
|
||||
@@ -77,4 +77,4 @@
|
||||
(fn [_ response]
|
||||
(let [exp (if (integer? max-age) max-age (inst-ms max-age))
|
||||
val (dm/fmt "max-age=%" (int (/ exp 1000.0)))]
|
||||
(update response ::rres/headers assoc "cache-control" val)))))
|
||||
(update response ::yrs/headers assoc "cache-control" val)))))
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.main :as-alias main]
|
||||
[app.setup.builtin-templates]
|
||||
[app.setup.keys :as keys]
|
||||
[app.setup.templates]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.nonce :as bn]
|
||||
[clojure.spec.alpha :as s]
|
||||
|
||||
72
backend/src/app/setup/builtin_templates.clj
Normal file
72
backend/src/app/setup/builtin_templates.clj
Normal file
@@ -0,0 +1,72 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.setup.builtin-templates
|
||||
"A service/module that is responsible for download, load & internally
|
||||
expose a set of builtin penpot file templates."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.http.client :as http]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.spec.alpha :as s]
|
||||
[datoteka.fs :as fs]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(declare download-all!)
|
||||
|
||||
(s/def ::id ::us/not-empty-string)
|
||||
(s/def ::name ::us/not-empty-string)
|
||||
(s/def ::thumbnail-uri ::us/not-empty-string)
|
||||
(s/def ::file-uri ::us/not-empty-string)
|
||||
(s/def ::path fs/path?)
|
||||
|
||||
(s/def ::template
|
||||
(s/keys :req-un [::id ::name ::thumbnail-uri ::file-uri]
|
||||
:opt-un [::path]))
|
||||
|
||||
(defmethod ig/pre-init-spec :app.setup/builtin-templates [_]
|
||||
(s/keys :req [::http/client]))
|
||||
|
||||
(defmethod ig/init-key :app.setup/builtin-templates
|
||||
[_ cfg]
|
||||
(let [presets (-> "app/onboarding.edn" io/resource slurp edn/read-string)]
|
||||
(l/info :hint "loading template files" :total (count presets))
|
||||
(let [result (download-all! cfg presets)]
|
||||
(us/conform (s/coll-of ::template) result))))
|
||||
|
||||
(defn- download-preset!
|
||||
[cfg {:keys [path file-uri] :as preset}]
|
||||
(let [response (http/req! cfg
|
||||
{:method :get
|
||||
:uri file-uri}
|
||||
{:response-type :input-stream
|
||||
:sync? true})]
|
||||
(us/verify! (= 200 (:status response)) "unexpected response found on fetching preset")
|
||||
(with-open [output (io/output-stream path)]
|
||||
(with-open [input (io/input-stream (:body response))]
|
||||
(io/copy input output)))))
|
||||
|
||||
(defn- download-all!
|
||||
"Download presets to the default directory, if preset is already
|
||||
downloaded, no action will be performed."
|
||||
[cfg presets]
|
||||
(let [dest (fs/join fs/*cwd* "builtin-templates")]
|
||||
(when-not (fs/exists? dest)
|
||||
(fs/create-dir dest))
|
||||
|
||||
(doall
|
||||
(map (fn [item]
|
||||
(let [path (fs/join dest (:id item))
|
||||
item (assoc item :path path)]
|
||||
(if (fs/exists? path)
|
||||
(l/trace :hint "template file already present" :id (:id item))
|
||||
(do
|
||||
(l/trace :hint "downloading template file" :id (:id item) :dest (str path))
|
||||
(download-preset! cfg item)))
|
||||
item))
|
||||
presets))))
|
||||
@@ -1,68 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.setup.templates
|
||||
"A service/module that is responsible for download, load & internally
|
||||
expose a set of builtin penpot file templates."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.http.client :as http]
|
||||
[app.setup :as-alias setup]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.java.io :as io]
|
||||
[datoteka.fs :as fs]
|
||||
[integrant.core :as ig]))
|
||||
|
||||
(def ^:private
|
||||
schema:template
|
||||
(sm/define
|
||||
[:map {:title "Template"}
|
||||
[:id ::sm/word-string]
|
||||
[:name ::sm/word-string]
|
||||
[:file-uri ::sm/word-string]]))
|
||||
|
||||
(def ^:private
|
||||
schema:templates
|
||||
(sm/define
|
||||
[:vector schema:template]))
|
||||
|
||||
(defmethod ig/init-key ::setup/templates
|
||||
[_ _]
|
||||
(let [templates (-> "app/onboarding.edn" io/resource slurp edn/read-string)
|
||||
dest (fs/join fs/*cwd* "builtin-templates")]
|
||||
|
||||
(dm/verify!
|
||||
"expected a valid templates file"
|
||||
(sm/check! schema:templates templates))
|
||||
|
||||
(doseq [{:keys [id path] :as template} templates]
|
||||
(let [path (or path (fs/join dest id))]
|
||||
(if (fs/exists? path)
|
||||
(l/dbg :hint "template file" :id id :state "present" :path (dm/str path))
|
||||
(l/dbg :hint "template file" :id id :state "absent"))))
|
||||
|
||||
templates))
|
||||
|
||||
(defn get-template-stream
|
||||
[cfg template-id]
|
||||
(when-let [template (d/seek #(= (:id %) template-id)
|
||||
(::setup/templates cfg))]
|
||||
(let [dest (fs/join fs/*cwd* "builtin-templates")
|
||||
path (or (:path template) (fs/join dest template-id))]
|
||||
(if (fs/exists? path)
|
||||
(io/input-stream path)
|
||||
(let [resp (http/req! cfg
|
||||
{:method :get :uri (:file-uri template)}
|
||||
{:response-type :input-stream :sync? true})]
|
||||
|
||||
(dm/verify!
|
||||
"unexpected response found on fetching template"
|
||||
(= 200 (:status resp)))
|
||||
|
||||
(io/input-stream (:body resp)))))))
|
||||
@@ -10,7 +10,7 @@
|
||||
[app.common.logging :as l]
|
||||
[app.common.spec :as us]
|
||||
[app.config :as cf]
|
||||
[app.srepl.cli]
|
||||
[app.srepl.ext]
|
||||
[app.srepl.main]
|
||||
[app.util.json :as json]
|
||||
[app.util.locks :as locks]
|
||||
@@ -36,9 +36,7 @@
|
||||
lock (locks/create)]
|
||||
(ccs/prepl *in*
|
||||
(fn [m]
|
||||
(binding [*out* out,
|
||||
*flush-on-newline* true,
|
||||
*print-readably* true]
|
||||
(binding [*out* out, *flush-on-newline* true, *print-readably* true]
|
||||
(locks/locking lock
|
||||
(println (json/encode-str m))))))))
|
||||
|
||||
@@ -46,10 +44,13 @@
|
||||
|
||||
(s/def ::port ::us/integer)
|
||||
(s/def ::host ::us/not-empty-string)
|
||||
(s/def ::flag #{:urepl-server :prepl-server})
|
||||
(s/def ::type #{::prepl ::urepl})
|
||||
(s/def ::key (s/tuple ::type ::us/keyword))
|
||||
|
||||
(defmethod ig/pre-init-spec ::server
|
||||
[_]
|
||||
(s/keys :req [::host ::port]))
|
||||
(s/keys :req [::flag ::host ::port]))
|
||||
|
||||
(defmethod ig/prep-key ::server
|
||||
[[type _] cfg]
|
||||
@@ -58,12 +59,6 @@
|
||||
(defmethod ig/init-key ::server
|
||||
[[type _] {:keys [::flag ::port ::host] :as cfg}]
|
||||
(when (contains? cf/flags flag)
|
||||
|
||||
(l/inf :hint "initializing repl server"
|
||||
:name (name type)
|
||||
:port port
|
||||
:host host)
|
||||
|
||||
(let [accept (case type
|
||||
::prepl 'app.srepl/json-repl
|
||||
::urepl 'app.srepl/user-repl)
|
||||
@@ -72,8 +67,14 @@
|
||||
:name (name type)
|
||||
:accept accept}]
|
||||
|
||||
(l/info :msg "initializing repl server"
|
||||
:name (name type)
|
||||
:port port
|
||||
:host host)
|
||||
|
||||
(ccs/start-server params)
|
||||
(assoc params :type type))))
|
||||
|
||||
params)))
|
||||
|
||||
(defmethod ig/halt-key! ::server
|
||||
[_ params]
|
||||
|
||||
@@ -1,296 +0,0 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.srepl.components-v2
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as pp]
|
||||
[app.db :as db]
|
||||
[app.features.components-v2 :as feat]
|
||||
[app.util.time :as dt]
|
||||
[cuerdas.core :as str]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.semaphore :as ps]
|
||||
[promesa.util :as pu]))
|
||||
|
||||
(defn- print-stats!
|
||||
[stats]
|
||||
(->> stats
|
||||
(into (sorted-map))
|
||||
(pp/pprint)))
|
||||
|
||||
(defn- report-progress-files
|
||||
[tpoint]
|
||||
(fn [_ _ oldv newv]
|
||||
(when (not= (:processed/files oldv)
|
||||
(:processed/files newv))
|
||||
(let [total (:total/files newv)
|
||||
completed (:processed/files newv)
|
||||
progress (/ (* completed 100.0) total)
|
||||
elapsed (tpoint)]
|
||||
(l/dbg :hint "progress"
|
||||
:completed (:processed/files newv)
|
||||
:total (:total/files newv)
|
||||
:progress (str (int progress) "%")
|
||||
:elapsed (dt/format-duration elapsed))))))
|
||||
|
||||
(defn- report-progress-teams
|
||||
[tpoint on-progress]
|
||||
(fn [_ _ oldv newv]
|
||||
(when (not= (:processed/teams oldv)
|
||||
(:processed/teams newv))
|
||||
(let [total (:total/teams newv)
|
||||
completed (:processed/teams newv)
|
||||
progress (/ (* completed 100.0) total)
|
||||
progress (str (int progress) "%")
|
||||
elapsed (dt/format-duration (tpoint))]
|
||||
|
||||
(when (fn? on-progress)
|
||||
(on-progress {:total total
|
||||
:elapsed elapsed
|
||||
:completed completed
|
||||
:progress progress}))
|
||||
|
||||
(l/dbg :hint "progress"
|
||||
:completed completed
|
||||
:progress progress
|
||||
:elapsed elapsed)))))
|
||||
|
||||
(defn- get-total-files
|
||||
[pool & {:keys [team-id]}]
|
||||
(if (some? team-id)
|
||||
(let [sql (str/concat
|
||||
"SELECT count(f.id) AS count FROM file AS f "
|
||||
" JOIN project AS p ON (p.id = f.project_id) "
|
||||
" WHERE p.team_id = ? AND f.deleted_at IS NULL "
|
||||
" AND p.deleted_at IS NULL")
|
||||
res (db/exec-one! pool [sql team-id])]
|
||||
(:count res))
|
||||
|
||||
(let [sql (str/concat
|
||||
"SELECT count(id) AS count FROM file "
|
||||
" WHERE deleted_at IS NULL")
|
||||
res (db/exec-one! pool [sql])]
|
||||
(:count res))))
|
||||
|
||||
(defn- get-total-teams
|
||||
[pool]
|
||||
(let [sql (str/concat
|
||||
"SELECT count(id) AS count FROM team "
|
||||
" WHERE deleted_at IS NULL")
|
||||
res (db/exec-one! pool [sql])]
|
||||
(:count res)))
|
||||
|
||||
(defn migrate-file!
|
||||
[system file-id & {:keys [rollback?] :or {rollback? true}}]
|
||||
|
||||
(l/dbg :hint "migrate:start")
|
||||
(let [tpoint (dt/tpoint)]
|
||||
(try
|
||||
(binding [feat/*stats* (atom {})]
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-file! file-id))
|
||||
|
||||
(-> (deref feat/*stats*)
|
||||
(assoc :elapsed (dt/format-duration (tpoint)))))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/wrn :hint "migrate:error" :cause cause))
|
||||
|
||||
(finally
|
||||
(let [elapsed (dt/format-duration (tpoint))]
|
||||
(l/dbg :hint "migrate:end" :elapsed elapsed))))))
|
||||
|
||||
(defn migrate-files!
|
||||
[{:keys [::db/pool] :as system}
|
||||
& {:keys [chunk-size max-jobs max-items start-at preset rollback? skip-on-error validate?]
|
||||
:or {chunk-size 10
|
||||
skip-on-error true
|
||||
max-jobs 10
|
||||
max-items Long/MAX_VALUE
|
||||
preset :shutdown-on-failure
|
||||
rollback? true
|
||||
validate? false}}]
|
||||
(letfn [(get-chunk [cursor]
|
||||
(let [sql (str/concat
|
||||
"SELECT id, created_at FROM file "
|
||||
" WHERE created_at < ? AND deleted_at IS NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! pool [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (seq rows)]))
|
||||
|
||||
(get-candidates []
|
||||
(->> (d/iteration get-chunk
|
||||
:vf second
|
||||
:kf first
|
||||
:initk (or start-at (dt/now)))
|
||||
(take max-items)
|
||||
(map :id)))]
|
||||
|
||||
(l/dbg :hint "migrate:start")
|
||||
(let [fsem (ps/create :permits max-jobs)
|
||||
total (get-total-files pool)
|
||||
stats (atom {:files/total total})
|
||||
tpoint (dt/tpoint)]
|
||||
|
||||
(add-watch stats :progress-report (report-progress-files tpoint))
|
||||
|
||||
(binding [feat/*stats* stats
|
||||
feat/*semaphore* fsem
|
||||
feat/*skip-on-error* skip-on-error]
|
||||
(try
|
||||
(pu/with-open [scope (px/structured-task-scope :preset preset :factory :virtual)]
|
||||
|
||||
(run! (fn [file-id]
|
||||
(ps/acquire! feat/*semaphore*)
|
||||
(px/submit! scope (fn []
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-file! file-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? (not skip-on-error))))))
|
||||
(get-candidates))
|
||||
|
||||
(p/await! scope))
|
||||
|
||||
(-> (deref feat/*stats*)
|
||||
(assoc :elapsed (dt/format-duration (tpoint))))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/dbg :hint "migrate:error" :cause cause))
|
||||
|
||||
(finally
|
||||
(let [elapsed (dt/format-duration (tpoint))]
|
||||
(l/dbg :hint "migrate:end" :elapsed elapsed))))))))
|
||||
|
||||
(defn migrate-team!
|
||||
[{:keys [::db/pool] :as system} team-id
|
||||
& {:keys [rollback? skip-on-error validate?]
|
||||
:or {rollback? true skip-on-error true validate? false}}]
|
||||
(l/dbg :hint "migrate:start")
|
||||
|
||||
(let [total (get-total-files pool :team-id team-id)
|
||||
stats (atom {:total/files total})
|
||||
tpoint (dt/tpoint)]
|
||||
|
||||
(add-watch stats :progress-report (report-progress-files tpoint))
|
||||
|
||||
(try
|
||||
(binding [feat/*stats* stats
|
||||
feat/*skip-on-error* skip-on-error]
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-team! team-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? (not skip-on-error)))
|
||||
|
||||
(print-stats!
|
||||
(-> (deref feat/*stats*)
|
||||
(dissoc :total/files)
|
||||
(assoc :elapsed (dt/format-duration (tpoint))))))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/dbg :hint "migrate:error" :cause cause))
|
||||
|
||||
(finally
|
||||
(let [elapsed (dt/format-duration (tpoint))]
|
||||
(l/dbg :hint "migrate:end" :elapsed elapsed))))))
|
||||
|
||||
(defn default-on-end
|
||||
[stats]
|
||||
(print-stats!
|
||||
(-> stats
|
||||
(update :elapsed/total dt/format-duration)
|
||||
(dissoc :total/teams))))
|
||||
|
||||
(defn migrate-teams!
|
||||
[{:keys [::db/pool] :as system}
|
||||
& {:keys [chunk-size max-jobs max-items start-at
|
||||
rollback? validate? preset skip-on-error
|
||||
max-time on-start on-progress on-error on-end]
|
||||
:or {chunk-size 10000
|
||||
validate? false
|
||||
rollback? true
|
||||
skip-on-error true
|
||||
on-end default-on-end
|
||||
preset :shutdown-on-failure
|
||||
max-jobs Integer/MAX_VALUE
|
||||
max-items Long/MAX_VALUE}}]
|
||||
|
||||
(letfn [(get-chunk [cursor]
|
||||
(let [sql (str/concat
|
||||
"SELECT id, created_at, features FROM team "
|
||||
" WHERE created_at < ? AND deleted_at IS NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! pool [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (seq rows)]))
|
||||
|
||||
(get-candidates []
|
||||
(->> (d/iteration get-chunk
|
||||
:vf second
|
||||
:kf first
|
||||
:initk (or start-at (dt/now)))
|
||||
(map #(update % :features db/decode-pgarray #{}))
|
||||
(remove #(contains? (:features %) "ephimeral/v2-migration"))
|
||||
(take max-items)
|
||||
(map :id)))
|
||||
|
||||
(migrate-team [team-id]
|
||||
(try
|
||||
(-> (assoc system ::db/rollback rollback?)
|
||||
(feat/migrate-team! team-id
|
||||
:validate? validate?
|
||||
:throw-on-validate? (not skip-on-error)))
|
||||
(catch Throwable cause
|
||||
(l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
|
||||
|
||||
(process-team [scope tpoint mtime team-id]
|
||||
(ps/acquire! feat/*semaphore*)
|
||||
(let [ts (tpoint)]
|
||||
(if (and mtime (neg? (compare mtime ts)))
|
||||
(l/inf :hint "max time constraint reached" :elapsed (dt/format-duration ts))
|
||||
(px/submit! scope (partial migrate-team team-id)))))]
|
||||
|
||||
(l/dbg :hint "migrate:start")
|
||||
|
||||
(let [sem (ps/create :permits max-jobs)
|
||||
total (get-total-teams pool)
|
||||
stats (atom {:total/teams (min total max-items)})
|
||||
tpoint (dt/tpoint)
|
||||
mtime (some-> max-time dt/duration)]
|
||||
|
||||
(when (fn? on-start)
|
||||
(on-start {:total total :rollback rollback?}))
|
||||
|
||||
(add-watch stats :progress-report (report-progress-teams tpoint on-progress))
|
||||
|
||||
(binding [feat/*stats* stats
|
||||
feat/*semaphore* sem
|
||||
feat/*skip-on-error* skip-on-error]
|
||||
(try
|
||||
(pu/with-open [scope (px/structured-task-scope :preset preset
|
||||
:factory :virtual)]
|
||||
(loop [candidates (get-candidates)]
|
||||
(when-let [team-id (first candidates)]
|
||||
(when (process-team scope tpoint mtime team-id)
|
||||
(recur (rest candidates)))))
|
||||
|
||||
(p/await! scope))
|
||||
|
||||
(when (fn? on-end)
|
||||
(-> (deref stats)
|
||||
(assoc :elapsed/total (tpoint))
|
||||
(on-end)))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/dbg :hint "migrate:error" :cause cause)
|
||||
(when (fn? on-error)
|
||||
(on-error cause)))
|
||||
|
||||
(finally
|
||||
(let [elapsed (dt/format-duration (tpoint))]
|
||||
(l/dbg :hint "migrate:end" :elapsed elapsed))))))))
|
||||
@@ -4,16 +4,14 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.srepl.cli
|
||||
(ns app.srepl.ext
|
||||
"PREPL API for external usage (CLI or ADMIN)"
|
||||
(:require
|
||||
[app.auth :as auth]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.db :as db]
|
||||
[app.main :as main]
|
||||
[app.rpc.commands.auth :as cmd.auth]
|
||||
[app.srepl.components-v2]
|
||||
[app.util.json :as json]
|
||||
[app.util.time :as dt]
|
||||
[cuerdas.core :as str]))
|
||||
@@ -23,18 +21,18 @@
|
||||
(or (deref (requiring-resolve 'app.main/system))
|
||||
(deref (requiring-resolve 'user/system))))
|
||||
|
||||
(defmulti ^:private exec-command ::cmd)
|
||||
(defmulti ^:private run-json-cmd* ::cmd)
|
||||
|
||||
(defn exec
|
||||
(defn run-json-cmd
|
||||
"Entry point with external tools integrations that uses PREPL
|
||||
interface for interacting with running penpot backend."
|
||||
[data]
|
||||
(let [data (json/decode data)]
|
||||
(-> {::cmd (keyword (:cmd data "default"))}
|
||||
(merge (:params data))
|
||||
(exec-command))))
|
||||
(let [data (json/decode data)
|
||||
params (merge {::cmd (keyword (:cmd data "default"))}
|
||||
(:params data))]
|
||||
(run-json-cmd* params)))
|
||||
|
||||
(defmethod exec-command :create-profile
|
||||
(defmethod run-json-cmd* :create-profile
|
||||
[{:keys [fullname email password is-active]
|
||||
:or {is-active true}}]
|
||||
(when-let [system (get-current-system)]
|
||||
@@ -48,7 +46,7 @@
|
||||
(->> (cmd.auth/create-profile! conn params)
|
||||
(cmd.auth/create-profile-rels! conn))))))
|
||||
|
||||
(defmethod exec-command :update-profile
|
||||
(defmethod run-json-cmd* :update-profile
|
||||
[{:keys [fullname email password is-active]}]
|
||||
(when-let [system (get-current-system)]
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
@@ -69,7 +67,7 @@
|
||||
{::db/return-keys? false})]
|
||||
(pos? (:next.jdbc/update-count res))))))))
|
||||
|
||||
(defmethod exec-command :delete-profile
|
||||
(defmethod run-json-cmd* :delete-profile
|
||||
[{:keys [email soft]}]
|
||||
(when-not email
|
||||
(ex/raise :type :assertion
|
||||
@@ -89,7 +87,7 @@
|
||||
{::db/return-keys? false}))]
|
||||
(pos? (:next.jdbc/update-count res))))))
|
||||
|
||||
(defmethod exec-command :search-profile
|
||||
(defmethod run-json-cmd* :search-profile
|
||||
[{:keys [email]}]
|
||||
(when-not email
|
||||
(ex/raise :type :assertion
|
||||
@@ -103,33 +101,11 @@
|
||||
" where email similar to ? order by created_at desc limit 100")]
|
||||
(db/exec! conn [sql email])))))
|
||||
|
||||
(defmethod exec-command :derive-password
|
||||
(defmethod run-json-cmd* :derive-password
|
||||
[{:keys [password]}]
|
||||
(auth/derive-password password))
|
||||
|
||||
(defmethod exec-command :migrate-v2
|
||||
[_]
|
||||
(letfn [(on-start [{:keys [total rollback]}]
|
||||
(println
|
||||
(str/ffmt "The components/v2 migration started (rollback:%, teams:%)"
|
||||
(if rollback "on" "off")
|
||||
total)))
|
||||
|
||||
(on-progress [{:keys [total elapsed progress completed]}]
|
||||
(println (str/ffmt "Progress % (total: %, completed: %, elapsed: %)"
|
||||
progress total completed elapsed)))
|
||||
(on-error [cause]
|
||||
(println "ERR:" (ex-message cause)))
|
||||
|
||||
(on-end [_]
|
||||
(println "Migration finished"))]
|
||||
(app.srepl.components-v2/migrate-teams! main/system
|
||||
:on-start on-start
|
||||
:on-error on-error
|
||||
:on-progress on-progress
|
||||
:on-end on-end)))
|
||||
|
||||
(defmethod exec-command :default
|
||||
(defmethod run-json-cmd* :default
|
||||
[{:keys [::cmd]}]
|
||||
(ex/raise :type :internal
|
||||
:code :not-implemented
|
||||
75
backend/src/app/srepl/fixes.clj
Normal file
75
backend/src/app/srepl/fixes.clj
Normal file
@@ -0,0 +1,75 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC
|
||||
|
||||
(ns app.srepl.fixes
|
||||
"A collection of adhoc fixes scripts."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.srepl.helpers :as h]))
|
||||
|
||||
(defn repair-orphaned-shapes
|
||||
"There are some shapes whose parent has been deleted. This function
|
||||
detects them and puts them as children of the root node."
|
||||
([data]
|
||||
(letfn [(is-orphan? [shape objects]
|
||||
(and (some? (:parent-id shape))
|
||||
(nil? (get objects (:parent-id shape)))))
|
||||
|
||||
(update-page [page]
|
||||
(let [objects (:objects page)
|
||||
orphans (into #{} (filter #(is-orphan? % objects)) (vals objects))]
|
||||
(if (seq orphans)
|
||||
(do
|
||||
(l/info :hint "found a file with orphans" :file-id (:id data) :broken-shapes (count orphans))
|
||||
(-> page
|
||||
(h/update-shapes (fn [shape]
|
||||
(if (contains? orphans shape)
|
||||
(assoc shape :parent-id uuid/zero)
|
||||
shape)))
|
||||
(update-in [:objects uuid/zero :shapes] into (map :id) orphans)))
|
||||
page)))]
|
||||
|
||||
(h/update-pages data update-page)))
|
||||
|
||||
;; special arity for to be called from h/analyze-files to search for
|
||||
;; files with possible issues
|
||||
|
||||
([file state]
|
||||
(repair-orphaned-shapes (:data file))
|
||||
(update state :total (fnil inc 0))))
|
||||
|
||||
(defn rename-layout-attrs
|
||||
([file]
|
||||
(let [found? (volatile! false)]
|
||||
(letfn [(update-shape
|
||||
[shape]
|
||||
(when (or (= (:layout-flex-dir shape) :reverse-row)
|
||||
(= (:layout-flex-dir shape) :reverse-column)
|
||||
(= (:layout-wrap-type shape) :no-wrap))
|
||||
(vreset! found? true))
|
||||
(cond-> shape
|
||||
(= (:layout-flex-dir shape) :reverse-row)
|
||||
(assoc :layout-flex-dir :row-reverse)
|
||||
(= (:layout-flex-dir shape) :reverse-column)
|
||||
(assoc :layout-flex-dir :column-reverse)
|
||||
(= (:layout-wrap-type shape) :no-wrap)
|
||||
(assoc :layout-wrap-type :nowrap)))
|
||||
|
||||
(update-page
|
||||
[page]
|
||||
(h/update-shapes page update-shape))]
|
||||
|
||||
(let [new-file (update file :data h/update-pages update-page)]
|
||||
(when @found?
|
||||
(l/info :hint "Found attrs to rename in file"
|
||||
:id (:id file)
|
||||
:name (:name file)))
|
||||
new-file))))
|
||||
|
||||
([file state]
|
||||
(rename-layout-attrs file)
|
||||
(update state :total (fnil inc 0))))
|
||||
@@ -6,27 +6,23 @@
|
||||
|
||||
(ns app.srepl.helpers
|
||||
"A main namespace for server repl."
|
||||
(:refer-clojure :exclude [parse-uuid])
|
||||
#_:clj-kondo/ignore
|
||||
(:require
|
||||
[app.auth :refer [derive-password]]
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.files.changes :as cpc]
|
||||
[app.common.files.migrations :as pmg]
|
||||
[app.common.files.repair :as repair]
|
||||
[app.common.files.validate :as validate]
|
||||
[app.common.files.features :as ffeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pages :as cp]
|
||||
[app.common.pages.migrations :as pmg]
|
||||
[app.common.pprint :refer [pprint]]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cfg]
|
||||
[app.db :as db]
|
||||
[app.db.sql :as sql]
|
||||
[app.features.fdata :as feat.fdata]
|
||||
[app.main :refer [system]]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.files-update :as files-update]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.objects-map :as omap]
|
||||
[app.util.pointer-map :as pmap]
|
||||
@@ -35,278 +31,124 @@
|
||||
[clojure.stacktrace :as strace]
|
||||
[clojure.walk :as walk]
|
||||
[cuerdas.core :as str]
|
||||
[expound.alpha :as expound]
|
||||
[promesa.core :as p]
|
||||
[promesa.exec :as px]
|
||||
[promesa.exec.csp :as sp]))
|
||||
[expound.alpha :as expound]))
|
||||
|
||||
(def ^:dynamic *system* nil)
|
||||
(def ^:dynamic *conn*)
|
||||
|
||||
(defn println!
|
||||
[& params]
|
||||
(locking println
|
||||
(apply println params)))
|
||||
|
||||
(defn parse-uuid
|
||||
[v]
|
||||
(if (uuid? v)
|
||||
v
|
||||
(d/parse-uuid v)))
|
||||
(defn reset-password!
|
||||
"Reset a password to a specific one for a concrete user or all users
|
||||
if email is `:all` keyword."
|
||||
[system & {:keys [email password] :or {password "123123"} :as params}]
|
||||
(us/verify! (contains? params :email) "`email` parameter is mandatory")
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [password (derive-password password)]
|
||||
(if (= email :all)
|
||||
(db/exec! conn ["update profile set password=?" password])
|
||||
(let [email (str/lower email)]
|
||||
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
|
||||
|
||||
(defn reset-file-data!
|
||||
"Hardcode replace of the data of one file."
|
||||
[system id data]
|
||||
(db/tx-run! system (fn [system]
|
||||
(db/update! system :file
|
||||
{:data data}
|
||||
{:id id}))))
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(db/update! conn :file
|
||||
{:data data}
|
||||
{:id id})))
|
||||
|
||||
(defn get-file
|
||||
"Get the migrated data of one file."
|
||||
[system id & {:keys [migrate?] :or {migrate? true}}]
|
||||
(db/run! system
|
||||
(fn [system]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(-> (files/get-file system id :migrate? migrate?)
|
||||
(update :data feat.fdata/process-pointers deref))))))
|
||||
|
||||
(defn validate
|
||||
"Validate structure, referencial integrity and semantic coherence of
|
||||
all contents of a file. Returns a list of errors."
|
||||
[system id]
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(let [id (if (string? id) (parse-uuid id) id)
|
||||
file (files/get-file system id)
|
||||
libs (->> (files/get-file-libraries conn id)
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(-> (files/get-file system id :migrate? false)
|
||||
(feat.fdata/process-pointers deref)
|
||||
(pmg/migrate-file))))))
|
||||
(d/index-by :id))]
|
||||
(validate/validate-file file libs))))))
|
||||
|
||||
(defn repair!
|
||||
"Repair the list of errors detected by validation."
|
||||
[system id]
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)
|
||||
pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(let [id (if (string? id) (parse-uuid id) id)
|
||||
file (files/get-file system id)
|
||||
libs (->> (files/get-file-libraries conn id)
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(-> (files/get-file system id :migrate? false)
|
||||
(feat.fdata/process-pointers deref)
|
||||
(pmg/migrate-file))))))
|
||||
(d/index-by :id))
|
||||
errors (validate/validate-file file libs)
|
||||
changes (repair/repair-file file libs errors)
|
||||
|
||||
file (-> file
|
||||
(update :revn inc)
|
||||
(update :data cpc/process-changes changes)
|
||||
(update :data blob/encode))]
|
||||
|
||||
(when (contains? (:features file) "fdata/pointer-map")
|
||||
(feat.fdata/persist-pointers! system id))
|
||||
|
||||
(db/update! conn :file
|
||||
{:revn (:revn file)
|
||||
:data (:data file)
|
||||
:data-backend nil
|
||||
:modified-at (dt/now)
|
||||
:has-media-trimmed false}
|
||||
{:id (:id file)})
|
||||
:repaired)))))
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(binding [pmap/*load-fn* (partial files/load-pointer conn id)]
|
||||
(-> (db/get-by-id conn :file id)
|
||||
(update :data blob/decode)
|
||||
(update :data pmg/migrate-data)
|
||||
(files/process-pointers deref)))))
|
||||
|
||||
(defn update-file!
|
||||
"Apply a function to the data of one file. Optionally save the changes or not.
|
||||
The function receives the decoded and migrated file data."
|
||||
[system & {:keys [update-fn id rollback? migrate? inc-revn?]
|
||||
:or {rollback? true migrate? true inc-revn? true}}]
|
||||
(letfn [(process-file [{:keys [::db/conn] :as system} {:keys [features] :as file}]
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)
|
||||
pmap/*load-fn* (partial feat.fdata/load-pointer system id)
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? features "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? features "fdata/objectd-map") omap/wrap identity)]
|
||||
[system & {:keys [update-fn id save? migrate? inc-revn?]
|
||||
:or {save? false migrate? true inc-revn? true}}]
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [file (-> (db/get-by-id conn :file id {::db/for-update? true})
|
||||
(update :features db/decode-pgarray #{}))]
|
||||
(binding [*conn* conn
|
||||
pmap/*tracked* (atom {})
|
||||
pmap/*load-fn* (partial files/load-pointer conn id)
|
||||
ffeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? (:features file) "storage/pointer-map") pmap/wrap identity)
|
||||
ffeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? (:features file) "storage/objectd-map") omap/wrap identity)]
|
||||
(let [file (-> file
|
||||
(update :data blob/decode)
|
||||
(cond-> migrate? (update :data pmg/migrate-data))
|
||||
(update-fn)
|
||||
(cond-> inc-revn? (update :revn inc)))]
|
||||
(when save?
|
||||
(let [features (db/create-array conn "text" (:features file))
|
||||
data (blob/encode (:data file))]
|
||||
(db/update! conn :file
|
||||
{:data data
|
||||
:revn (:revn file)
|
||||
:features features}
|
||||
{:id id})
|
||||
|
||||
(let [file (cond-> (update-fn file)
|
||||
inc-revn? (update :revn inc))
|
||||
features (db/create-array conn "text" (:features file))
|
||||
data (blob/encode (:data file))]
|
||||
(when (contains? (:features file) "storage/pointer-map")
|
||||
(files/persist-pointers! conn id))))
|
||||
|
||||
(db/update! conn :file
|
||||
{:data data
|
||||
:revn (:revn file)
|
||||
:features features}
|
||||
{:id id}))
|
||||
(dissoc file :data))))))
|
||||
|
||||
(when (contains? (:features file) "fdata/pointer-map")
|
||||
(feat.fdata/persist-pointers! system id))
|
||||
|
||||
(dissoc file :data)))]
|
||||
|
||||
(db/tx-run! system
|
||||
(fn [system]
|
||||
(binding [*system* system]
|
||||
(try
|
||||
(->> (files/get-file system id :migrate? migrate?)
|
||||
(process-file system))
|
||||
(finally
|
||||
(when rollback?
|
||||
(db/rollback! system)))))))))
|
||||
(def ^:private sql:retrieve-files-chunk
|
||||
"SELECT id, name, created_at, data FROM file
|
||||
WHERE created_at < ? AND deleted_at is NULL
|
||||
ORDER BY created_at desc LIMIT ?")
|
||||
|
||||
(defn analyze-files
|
||||
"Apply a function to all files in the database, reading them in
|
||||
batches. Do not change data.
|
||||
|
||||
The `on-file` parameter should be a function that receives the file
|
||||
and the previous state and returns the new state.
|
||||
|
||||
Emits rollback at the end of operation."
|
||||
[system & {:keys [chunk-size max-items start-at on-file on-error on-end on-init with-libraries?]
|
||||
and the previous state and returns the new state."
|
||||
[system & {:keys [chunk-size max-items start-at on-file on-error on-end]
|
||||
:or {chunk-size 10 max-items Long/MAX_VALUE}}]
|
||||
(letfn [(get-chunk [conn cursor]
|
||||
(let [sql (str "SELECT id, created_at FROM file "
|
||||
" WHERE created_at < ? AND deleted_at is NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! conn [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (map :id rows)]))
|
||||
(let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (seq rows)]))
|
||||
|
||||
(get-candidates [conn]
|
||||
(->> (d/iteration (partial get-chunk conn)
|
||||
:vf second
|
||||
:kf first
|
||||
:initk (or start-at (dt/now)))
|
||||
(take max-items)))
|
||||
(take max-items)
|
||||
(map #(update % :data blob/decode))))
|
||||
|
||||
(on-error* [cause file]
|
||||
(on-error* [file cause]
|
||||
(println "unexpected exception happened on processing file: " (:id file))
|
||||
(strace/print-stack-trace cause))
|
||||
(strace/print-stack-trace cause))]
|
||||
|
||||
(process-file [{:keys [::db/conn] :as system} file-id]
|
||||
(let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)]
|
||||
(-> (files/get-file system file-id)
|
||||
(update :data feat.fdata/process-pointers deref)))
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(loop [state {}
|
||||
files (get-candidates conn)]
|
||||
(if-let [file (first files)]
|
||||
(let [state' (try
|
||||
(on-file file state)
|
||||
(catch Throwable cause
|
||||
(let [on-error (or on-error on-error*)]
|
||||
(on-error file cause))))]
|
||||
(recur (or state' state) (rest files)))
|
||||
|
||||
libs (when with-libraries?
|
||||
(->> (files/get-file-libraries conn file-id)
|
||||
(into [file] (map (fn [{:keys [id]}]
|
||||
(binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)]
|
||||
(-> (files/get-file system id)
|
||||
(update :data feat.fdata/process-pointers deref))))))
|
||||
(d/index-by :id)))]
|
||||
(try
|
||||
(if with-libraries?
|
||||
(on-file file libs)
|
||||
(on-file file))
|
||||
(catch Throwable cause
|
||||
((or on-error on-error*) cause file)))))]
|
||||
(if (fn? on-end)
|
||||
(on-end state)
|
||||
state))))))
|
||||
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn] :as system}]
|
||||
(try
|
||||
(binding [*system* system]
|
||||
(when (fn? on-init) (on-init))
|
||||
(run! (partial process-file system) (get-candidates conn)))
|
||||
(finally
|
||||
(when (fn? on-end)
|
||||
(ex/ignoring (on-end)))
|
||||
(db/rollback! system)))))))
|
||||
(defn update-pages
|
||||
"Apply a function to all pages of one file. The function receives a page and returns an updated page."
|
||||
[data f]
|
||||
(update data :pages-index update-vals f))
|
||||
|
||||
(defn process-files!
|
||||
"Apply a function to all files in the database, reading them in
|
||||
batches."
|
||||
|
||||
[system & {:keys [chunk-size
|
||||
max-items
|
||||
workers
|
||||
start-at
|
||||
on-file
|
||||
on-error
|
||||
on-end
|
||||
on-init
|
||||
rollback?]
|
||||
:or {chunk-size 10
|
||||
max-items Long/MAX_VALUE
|
||||
workers 1
|
||||
rollback? true}}]
|
||||
(letfn [(get-chunk [conn cursor]
|
||||
(let [sql (str "SELECT id, created_at FROM file "
|
||||
" WHERE created_at < ? AND deleted_at is NULL "
|
||||
" ORDER BY created_at desc LIMIT ?")
|
||||
rows (db/exec! conn [sql cursor chunk-size])]
|
||||
[(some->> rows peek :created-at) (map :id rows)]))
|
||||
|
||||
(get-candidates [conn]
|
||||
(->> (d/iteration (partial get-chunk conn)
|
||||
:vf second
|
||||
:kf first
|
||||
:initk (or start-at (dt/now)))
|
||||
(take max-items)))
|
||||
|
||||
(on-error* [cause file]
|
||||
(println! "unexpected exception happened on processing file: " (:id file))
|
||||
(strace/print-stack-trace cause))
|
||||
|
||||
(process-file [system file-id]
|
||||
(try
|
||||
(let [{:keys [features] :as file} (files/get-file system file-id)]
|
||||
(binding [pmap/*tracked* (pmap/create-tracked)
|
||||
pmap/*load-fn* (partial feat.fdata/load-pointer system file-id)
|
||||
cfeat/*wrap-with-pointer-map-fn*
|
||||
(if (contains? features "fdata/pointer-map") pmap/wrap identity)
|
||||
cfeat/*wrap-with-objects-map-fn*
|
||||
(if (contains? features "fdata/objectd-map") omap/wrap identity)]
|
||||
|
||||
(on-file file)
|
||||
|
||||
(when (contains? features "fdata/pointer-map")
|
||||
(feat.fdata/persist-pointers! system file-id))))
|
||||
|
||||
(catch Throwable cause
|
||||
((or on-error on-error*) cause file-id))))
|
||||
|
||||
(run-worker [in index]
|
||||
(db/tx-run! system
|
||||
(fn [system]
|
||||
(binding [*system* system]
|
||||
(loop [i 0]
|
||||
(when-let [file-id (sp/take! in)]
|
||||
(println! "=> worker: index:" index "| loop:" i "| file:" (str file-id) "|" (px/get-name))
|
||||
(process-file system file-id)
|
||||
(recur (inc i)))))
|
||||
|
||||
(when rollback?
|
||||
(db/rollback! system)))))
|
||||
|
||||
(run-producer [input]
|
||||
(db/tx-run! system (fn [{:keys [::db/conn]}]
|
||||
(doseq [file-id (get-candidates conn)]
|
||||
(println! "=> producer:" file-id "|" (px/get-name))
|
||||
(sp/put! input file-id))
|
||||
(sp/close! input))))]
|
||||
|
||||
(when (fn? on-init) (on-init))
|
||||
|
||||
(let [input (sp/chan :buf chunk-size)
|
||||
producer (px/thread
|
||||
{:name "penpot/srepl/producer"}
|
||||
(run-producer input))
|
||||
threads (->> (range workers)
|
||||
(map (fn [index]
|
||||
(px/thread
|
||||
{:name (str "penpot/srepl/worker/" index)}
|
||||
(run-worker input index))))
|
||||
(cons producer)
|
||||
(doall))]
|
||||
|
||||
(run! p/await! threads)
|
||||
(when (fn? on-end) (on-end)))))
|
||||
(defn update-shapes
|
||||
"Apply a function to all shapes of one page The function receives a shape and returns an updated shape"
|
||||
[page f]
|
||||
(update page :objects update-vals f))
|
||||
|
||||
@@ -8,32 +8,20 @@
|
||||
"A collection of adhoc fixes scripts."
|
||||
#_:clj-kondo/ignore
|
||||
(:require
|
||||
[app.auth :refer [derive-password]]
|
||||
[app.common.data :as d]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.features :as cfeat]
|
||||
[app.common.logging :as l]
|
||||
[app.common.pprint :as p]
|
||||
[app.common.spec :as us]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.features.fdata :as features.fdata]
|
||||
[app.main :as main]
|
||||
[app.msgbus :as mbus]
|
||||
[app.rpc.commands.auth :as auth]
|
||||
[app.rpc.commands.files-snapshot :as fsnap]
|
||||
[app.rpc.commands.profile :as profile]
|
||||
[app.srepl.cli :as cli]
|
||||
[app.srepl.fixes :as f]
|
||||
[app.srepl.helpers :as h]
|
||||
[app.storage :as sto]
|
||||
[app.util.blob :as blob]
|
||||
[app.util.objects-map :as omap]
|
||||
[app.util.pointer-map :as pmap]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.pprint :refer [pprint print-table]]
|
||||
[clojure.tools.namespace.repl :as repl]
|
||||
[clojure.pprint :refer [pprint]]
|
||||
[cuerdas.core :as str]))
|
||||
|
||||
(defn print-available-tasks
|
||||
@@ -113,71 +101,44 @@
|
||||
(db/delete! conn :http-session {:profile-id (:id profile)})
|
||||
:blocked))))
|
||||
|
||||
(defn reset-password!
|
||||
"Reset a password to a specific one for a concrete user or all users
|
||||
if email is `:all` keyword."
|
||||
[system & {:keys [email password] :or {password "123123"} :as params}]
|
||||
(us/verify! (contains? params :email) "`email` parameter is mandatory")
|
||||
(db/with-atomic [conn (:app.db/pool system)]
|
||||
(let [password (derive-password password)]
|
||||
(if (= email :all)
|
||||
(db/exec! conn ["update profile set password=?" password])
|
||||
(let [email (str/lower email)]
|
||||
(db/exec! conn ["update profile set password=? where email=?" password email]))))))
|
||||
|
||||
(defn enable-objects-map-feature-on-file!
|
||||
[system & {:keys [save? id]}]
|
||||
(h/update-file! system
|
||||
:id id
|
||||
:update-fn features.fdata/enable-objects-map
|
||||
:save? save?))
|
||||
(letfn [(update-file [{:keys [features] :as file}]
|
||||
(if (contains? features "storage/objects-map")
|
||||
file
|
||||
(-> file
|
||||
(update :data migrate)
|
||||
(update :features conj "storage/objects-map"))))
|
||||
|
||||
(migrate [data]
|
||||
(-> data
|
||||
(update :pages-index update-vals #(update % :objects omap/wrap))
|
||||
(update :components update-vals #(update % :objects omap/wrap))))]
|
||||
|
||||
(h/update-file! system
|
||||
:id id
|
||||
:update-fn update-file
|
||||
:save? save?)))
|
||||
|
||||
(defn enable-pointer-map-feature-on-file!
|
||||
[system & {:keys [save? id]}]
|
||||
(h/update-file! system
|
||||
:id id
|
||||
:update-fn features.fdata/enable-pointer-map
|
||||
:save? save?))
|
||||
(letfn [(update-file [{:keys [features] :as file}]
|
||||
(if (contains? features "storage/pointer-map")
|
||||
file
|
||||
(-> file
|
||||
(update :data migrate)
|
||||
(update :features conj "storage/pointer-map"))))
|
||||
|
||||
(defn enable-team-feature!
|
||||
[system team-id feature]
|
||||
(dm/verify!
|
||||
"feature should be supported"
|
||||
(contains? cfeat/supported-features feature))
|
||||
(migrate [data]
|
||||
(-> data
|
||||
(update :pages-index update-vals pmap/wrap)
|
||||
(update :components pmap/wrap)))]
|
||||
|
||||
(let [team-id (if (string? team-id)
|
||||
(parse-uuid team-id)
|
||||
team-id)]
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(let [team (-> (db/get conn :team {:id team-id})
|
||||
(update :features db/decode-pgarray #{}))
|
||||
features (conj (:features team) feature)]
|
||||
(when (not= features (:features team))
|
||||
(db/update! conn :team
|
||||
{:features (db/create-array conn "text" features)}
|
||||
{:id team-id})
|
||||
:enabled))))))
|
||||
|
||||
(defn disable-team-feature!
|
||||
[system team-id feature]
|
||||
(dm/verify!
|
||||
"feature should be supported"
|
||||
(contains? cfeat/supported-features feature))
|
||||
|
||||
(let [team-id (if (string? team-id)
|
||||
(parse-uuid team-id)
|
||||
team-id)]
|
||||
(db/tx-run! system
|
||||
(fn [{:keys [::db/conn]}]
|
||||
(let [team (-> (db/get conn :team {:id team-id})
|
||||
(update :features db/decode-pgarray #{}))
|
||||
features (disj (:features team) feature)]
|
||||
(when (not= features (:features team))
|
||||
(db/update! conn :team
|
||||
{:features (db/create-array conn "text" features)}
|
||||
{:id team-id})
|
||||
:disabled))))))
|
||||
(h/update-file! system
|
||||
:id id
|
||||
:update-fn update-file
|
||||
:save? save?)))
|
||||
|
||||
(defn enable-storage-features-on-file!
|
||||
[system & {:as params}]
|
||||
@@ -203,132 +164,3 @@
|
||||
(alter-var-root var (fn [f]
|
||||
(or (::original (meta f)) f))))
|
||||
|
||||
(defn take-file-snapshot!
|
||||
"An internal helper that persist the file snapshot using non-gc
|
||||
collectable file-changes entry."
|
||||
[system & {:keys [file-id label]}]
|
||||
(let [file-id (h/parse-uuid file-id)]
|
||||
(db/tx-run! system fsnap/take-file-snapshot! {:file-id file-id :label label})))
|
||||
|
||||
(defn restore-file-snapshot!
|
||||
[system & {:keys [file-id id]}]
|
||||
(db/tx-run! system
|
||||
(fn [cfg]
|
||||
(let [file-id (h/parse-uuid file-id)
|
||||
id (h/parse-uuid id)]
|
||||
|
||||
(if (and (uuid? id) (uuid? file-id))
|
||||
(fsnap/restore-file-snapshot! cfg {:id id :file-id file-id})
|
||||
(println "=> invalid parameters"))))))
|
||||
|
||||
|
||||
(defn list-file-snapshots!
|
||||
[system & {:keys [file-id limit]}]
|
||||
(db/tx-run! system (fn [system]
|
||||
(let [params {:file-id (h/parse-uuid file-id)
|
||||
:limit limit}]
|
||||
(->> (fsnap/get-file-snapshots system (d/without-nils params))
|
||||
(print-table [:id :revn :created-at :label]))))))
|
||||
|
||||
(defn notify!
|
||||
[{:keys [::mbus/msgbus ::db/pool]} & {:keys [dest code message level]
|
||||
:or {code :generic level :info}
|
||||
:as params}]
|
||||
(dm/verify!
|
||||
["invalid level %" level]
|
||||
(contains? #{:success :error :info :warning} level))
|
||||
|
||||
(dm/verify!
|
||||
["invalid code: %" code]
|
||||
(contains? #{:generic :upgrade-version} code))
|
||||
|
||||
(letfn [(send [dest]
|
||||
(l/inf :hint "sending notification" :dest (str dest))
|
||||
(let [message {:type :notification
|
||||
:code code
|
||||
:level level
|
||||
:version (:full cf/version)
|
||||
:subs-id dest
|
||||
:message message}
|
||||
message (->> (dissoc params :dest :code :message :level)
|
||||
(merge message))]
|
||||
(mbus/pub! msgbus
|
||||
:topic (str dest)
|
||||
:message message)))
|
||||
|
||||
(resolve-profile [email]
|
||||
(some-> (db/get* pool :profile {:email (str/lower email)} {:columns [:id]}) :id vector))
|
||||
|
||||
(resolve-team [team-id]
|
||||
(->> (db/query pool :team-profile-rel
|
||||
{:team-id team-id}
|
||||
{:columns [:profile-id]})
|
||||
(map :profile-id)))
|
||||
|
||||
(parse-uuid [v]
|
||||
(if (uuid? v)
|
||||
v
|
||||
(d/parse-uuid v)))
|
||||
|
||||
(resolve-dest [dest]
|
||||
(cond
|
||||
(uuid? dest)
|
||||
[dest]
|
||||
|
||||
(string? dest)
|
||||
(some-> dest parse-uuid resolve-dest)
|
||||
|
||||
(nil? dest)
|
||||
(resolve-dest uuid/zero)
|
||||
|
||||
(map? dest)
|
||||
(sequence (comp
|
||||
(map vec)
|
||||
(mapcat resolve-dest))
|
||||
dest)
|
||||
|
||||
(and (coll? dest)
|
||||
(every? coll? dest))
|
||||
(sequence (comp
|
||||
(map vec)
|
||||
(mapcat resolve-dest))
|
||||
dest)
|
||||
|
||||
(vector? dest)
|
||||
(let [[op param] dest]
|
||||
(cond
|
||||
(= op :email)
|
||||
(cond
|
||||
(and (coll? param)
|
||||
(every? string? param))
|
||||
(sequence (comp
|
||||
(keep resolve-profile)
|
||||
(mapcat identity))
|
||||
param)
|
||||
|
||||
(string? param)
|
||||
(resolve-profile param))
|
||||
|
||||
(= op :team-id)
|
||||
(cond
|
||||
(coll? param)
|
||||
(sequence (comp
|
||||
(mapcat resolve-team)
|
||||
(keep parse-uuid))
|
||||
param)
|
||||
|
||||
(uuid? param)
|
||||
(resolve-team param)
|
||||
|
||||
(string? param)
|
||||
(some-> param parse-uuid resolve-team))
|
||||
|
||||
(= op :profile-id)
|
||||
(if (coll? param)
|
||||
(sequence (keep parse-uuid) param)
|
||||
(resolve-dest param))))))]
|
||||
|
||||
(->> (resolve-dest dest)
|
||||
(filter some?)
|
||||
(into #{})
|
||||
(run! send))))
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
[app.storage.impl :as impl]
|
||||
[app.storage.s3 :as ss3]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.spec.alpha :as s]
|
||||
[datoteka.fs :as fs]
|
||||
[integrant.core :as ig]
|
||||
@@ -39,7 +40,7 @@
|
||||
:fs ::sfs/backend))))
|
||||
|
||||
(defmethod ig/pre-init-spec ::storage [_]
|
||||
(s/keys :req [::db/pool ::backends]))
|
||||
(s/keys :req [::db/pool ::wrk/executor ::backends]))
|
||||
|
||||
(defmethod ig/init-key ::storage
|
||||
[_ {:keys [::backends ::db/pool] :as cfg}]
|
||||
@@ -250,59 +251,53 @@
|
||||
|
||||
(defmethod ig/init-key ::gc-deleted-task
|
||||
[_ {:keys [::db/pool ::storage ::min-age]}]
|
||||
(letfn [(get-to-delete-chunk [cursor]
|
||||
(let [sql (str "select s.* "
|
||||
" from storage_object as s "
|
||||
" where s.deleted_at is not null "
|
||||
" and s.deleted_at < ? "
|
||||
" order by s.deleted_at desc "
|
||||
" limit 25")
|
||||
rows (db/exec! pool [sql cursor])]
|
||||
[(some-> rows peek :deleted-at)
|
||||
(letfn [(retrieve-deleted-objects-chunk [conn min-age cursor]
|
||||
(let [min-age (db/interval min-age)
|
||||
rows (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
|
||||
[(some-> rows peek :created-at)
|
||||
(some->> (seq rows) (d/group-by #(-> % :backend keyword) :id #{}) seq)]))
|
||||
|
||||
(get-to-delete-chunks [min-age]
|
||||
(d/iteration get-to-delete-chunk
|
||||
:initk (dt/minus (dt/now) min-age)
|
||||
(retrieve-deleted-objects [conn min-age]
|
||||
(d/iteration (partial retrieve-deleted-objects-chunk conn min-age)
|
||||
:initk (dt/now)
|
||||
:vf second
|
||||
:kf first))
|
||||
|
||||
(delete-in-bulk! [backend-id ids]
|
||||
(try
|
||||
(db/with-atomic [conn pool]
|
||||
(let [sql "delete from storage_object where id = ANY(?)"
|
||||
ids' (db/create-array conn "uuid" ids)
|
||||
(delete-in-bulk [backend-id ids]
|
||||
(let [backend (impl/resolve-backend storage backend-id)]
|
||||
|
||||
total (-> (db/exec-one! conn [sql ids'])
|
||||
(db/get-update-count))]
|
||||
(doseq [id ids]
|
||||
(l/debug :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
|
||||
|
||||
(-> (impl/resolve-backend storage backend-id)
|
||||
(impl/del-objects-in-bulk ids))
|
||||
|
||||
(doseq [id ids]
|
||||
(l/dbg :hint "gc-deleted: permanently delete storage object" :backend backend-id :id id))
|
||||
|
||||
total))
|
||||
|
||||
(catch Throwable cause
|
||||
(l/err :hint "gc-deleted: unexpected error on bulk deletion"
|
||||
:ids (vec ids)
|
||||
:cause cause)
|
||||
0)))]
|
||||
(impl/del-objects-in-bulk backend ids)))]
|
||||
|
||||
(fn [params]
|
||||
(let [min-age (or (some-> params :min-age dt/duration) min-age)]
|
||||
(loop [total 0
|
||||
chunks (get-to-delete-chunks min-age)]
|
||||
(if-let [[backend-id ids] (first chunks)]
|
||||
(let [deleted (delete-in-bulk! backend-id ids)]
|
||||
(recur (+ total deleted)
|
||||
(rest chunks)))
|
||||
(do
|
||||
(l/inf :hint "gc-deleted: task finished"
|
||||
:min-age (dt/format-duration min-age)
|
||||
:total total)
|
||||
{:deleted total})))))))
|
||||
(let [min-age (or (:min-age params) min-age)]
|
||||
(db/with-atomic [conn pool]
|
||||
(loop [total 0
|
||||
groups (retrieve-deleted-objects conn min-age)]
|
||||
(if-let [[backend-id ids] (first groups)]
|
||||
(do
|
||||
(delete-in-bulk backend-id ids)
|
||||
(recur (+ total (count ids))
|
||||
(rest groups)))
|
||||
(do
|
||||
(l/info :hint "gc-deleted: task finished" :min-age (dt/format-duration min-age) :total total)
|
||||
{:deleted total}))))))))
|
||||
|
||||
(def sql:retrieve-deleted-objects-chunk
|
||||
"with items_part as (
|
||||
select s.id
|
||||
from storage_object as s
|
||||
where s.deleted_at is not null
|
||||
and s.deleted_at < (now() - ?::interval)
|
||||
and s.created_at < ?
|
||||
order by s.created_at desc
|
||||
limit 25
|
||||
)
|
||||
delete from storage_object
|
||||
where id in (select id from items_part)
|
||||
returning *;")
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Garbage Collection: Analyze touched objects
|
||||
@@ -321,11 +316,10 @@
|
||||
;; and the object is still valid) or deleted (no more references to
|
||||
;; this object so is ready to be deleted).
|
||||
|
||||
(declare sql:retrieve-file-media-object-nrefs)
|
||||
(declare sql:retrieve-file-object-thumbnail-nrefs)
|
||||
(declare sql:retrieve-profile-nrefs)
|
||||
(declare sql:retrieve-team-font-variant-nrefs)
|
||||
(declare sql:retrieve-touched-objects-chunk)
|
||||
(declare sql:retrieve-file-media-object-nrefs)
|
||||
(declare sql:retrieve-team-font-variant-nrefs)
|
||||
(declare sql:retrieve-profile-nrefs)
|
||||
|
||||
(defmethod ig/pre-init-spec ::gc-touched-task [_]
|
||||
(s/keys :req [::db/pool]))
|
||||
@@ -341,9 +335,6 @@
|
||||
(get-profile-nrefs [conn id]
|
||||
(-> (db/exec-one! conn [sql:retrieve-profile-nrefs id id]) :nrefs))
|
||||
|
||||
(get-file-object-thumbnails [conn id]
|
||||
(-> (db/exec-one! conn [sql:retrieve-file-object-thumbnail-nrefs id]) :nrefs))
|
||||
|
||||
(mark-freeze-in-bulk [conn ids]
|
||||
(db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)"
|
||||
(db/create-array conn "uuid" ids)]))
|
||||
@@ -403,7 +394,8 @@
|
||||
(do
|
||||
(some->> (seq to-freeze) (mark-freeze-in-bulk conn))
|
||||
(some->> (seq to-delete) (mark-delete-in-bulk conn))
|
||||
[(count to-freeze) (count to-delete)]))))]
|
||||
[(count to-freeze) (count to-delete)]))))
|
||||
]
|
||||
|
||||
(fn [_]
|
||||
(db/with-atomic [conn pool]
|
||||
@@ -412,10 +404,9 @@
|
||||
groups (retrieve-touched conn)]
|
||||
(if-let [[bucket ids] (first groups)]
|
||||
(let [[f d] (case bucket
|
||||
"file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket)
|
||||
"team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket)
|
||||
"file-object-thumbnail" (process-objects! conn get-file-object-thumbnails ids bucket)
|
||||
"profile" (process-objects! conn get-profile-nrefs ids bucket)
|
||||
"file-media-object" (process-objects! conn get-file-media-object-nrefs ids bucket)
|
||||
"team-font-variant" (process-objects! conn get-team-font-variant-nrefs ids bucket)
|
||||
"profile" (process-objects! conn get-profile-nrefs ids bucket)
|
||||
(ex/raise :type :internal
|
||||
:code :unexpected-unknown-reference
|
||||
:hint (dm/fmt "unknown reference %" bucket)))]
|
||||
@@ -438,9 +429,6 @@
|
||||
"select ((select count(*) from file_media_object where media_id = ?) +
|
||||
(select count(*) from file_media_object where thumbnail_id = ?)) as nrefs")
|
||||
|
||||
(def sql:retrieve-file-object-thumbnail-nrefs
|
||||
"select (select count(*) from file_tagged_object_thumbnail where media_id = ?) as nrefs")
|
||||
|
||||
(def sql:retrieve-team-font-variant-nrefs
|
||||
"select ((select count(*) from team_font_variant where woff1_file_id = ?) +
|
||||
(select count(*) from team_font_variant where woff2_file_id = ?) +
|
||||
|
||||
@@ -18,8 +18,8 @@
|
||||
[datoteka.io :as io]
|
||||
[integrant.core :as ig])
|
||||
(:import
|
||||
java.nio.file.Files
|
||||
java.nio.file.Path))
|
||||
java.nio.file.Path
|
||||
java.nio.file.Files))
|
||||
|
||||
;; --- BACKEND INIT
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
[app.common.exceptions :as ex]
|
||||
[app.db :as-alias db]
|
||||
[app.storage :as-alias sto]
|
||||
[app.worker :as-alias wrk]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.hash :as bh]
|
||||
[clojure.java.io :as jio]
|
||||
@@ -200,7 +201,7 @@
|
||||
(str "blake2b:" result)))
|
||||
|
||||
(defn resolve-backend
|
||||
[{:keys [::db/pool] :as storage} backend-id]
|
||||
[{:keys [::db/pool ::wrk/executor] :as storage} backend-id]
|
||||
(let [backend (get-in storage [::sto/backends backend-id])]
|
||||
(when-not backend
|
||||
(ex/raise :type :internal
|
||||
@@ -208,6 +209,7 @@
|
||||
:hint (dm/fmt "backend '%' not configured" backend-id)))
|
||||
(-> backend
|
||||
(assoc ::sto/id backend-id)
|
||||
(assoc ::wrk/executor executor)
|
||||
(assoc ::db/pool pool))))
|
||||
|
||||
(defrecord StorageObject [id size created-at expired-at touched-at backend])
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
[app.storage.impl :as impl]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.time :as dt]
|
||||
[app.worker :as wrk]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.spec.alpha :as s]
|
||||
[datoteka.fs :as fs]
|
||||
@@ -39,6 +40,7 @@
|
||||
software.amazon.awssdk.core.async.AsyncRequestBody
|
||||
software.amazon.awssdk.core.async.AsyncResponseTransformer
|
||||
software.amazon.awssdk.core.client.config.ClientAsyncConfiguration
|
||||
software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption
|
||||
software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient
|
||||
software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup
|
||||
software.amazon.awssdk.regions.Region
|
||||
@@ -75,10 +77,9 @@
|
||||
(s/def ::bucket ::us/string)
|
||||
(s/def ::prefix ::us/string)
|
||||
(s/def ::endpoint ::us/string)
|
||||
(s/def ::io-threads ::us/integer)
|
||||
|
||||
(defmethod ig/pre-init-spec ::backend [_]
|
||||
(s/keys :opt [::region ::bucket ::prefix ::endpoint ::io-threads]))
|
||||
(s/keys :opt [::region ::bucket ::prefix ::endpoint ::wrk/executor]))
|
||||
|
||||
(defmethod ig/prep-key ::backend
|
||||
[_ {:keys [::prefix ::region] :as cfg}]
|
||||
@@ -113,7 +114,8 @@
|
||||
::client
|
||||
::presigner]
|
||||
:opt [::prefix
|
||||
::sto/id]))
|
||||
::sto/id
|
||||
::wrk/executor]))
|
||||
|
||||
;; --- API IMPL
|
||||
|
||||
@@ -159,6 +161,7 @@
|
||||
|
||||
;; --- HELPERS
|
||||
|
||||
(def default-eventloop-threads 4)
|
||||
(def default-timeout
|
||||
(dt/duration {:seconds 30}))
|
||||
|
||||
@@ -168,18 +171,18 @@
|
||||
(Region/of (name region)))
|
||||
|
||||
(defn- build-s3-client
|
||||
[{:keys [::region ::endpoint ::io-threads]}]
|
||||
[{:keys [::region ::endpoint ::wrk/executor]}]
|
||||
(let [aconfig (-> (ClientAsyncConfiguration/builder)
|
||||
(.advancedOption SdkAdvancedAsyncClientOption/FUTURE_COMPLETION_EXECUTOR executor)
|
||||
(.build))
|
||||
|
||||
sconfig (-> (S3Configuration/builder)
|
||||
(cond-> (some? endpoint) (.pathStyleAccessEnabled true))
|
||||
(.build))
|
||||
|
||||
thr-num (or io-threads (min 16 (px/get-available-processors)))
|
||||
hclient (-> (NettyNioAsyncHttpClient/builder)
|
||||
(.eventLoopGroupBuilder (-> (SdkEventLoopGroup/builder)
|
||||
(.numberOfThreads (int thr-num))))
|
||||
(.numberOfThreads (int default-eventloop-threads))))
|
||||
(.connectionAcquisitionTimeout default-timeout)
|
||||
(.connectionTimeout default-timeout)
|
||||
(.readTimeout default-timeout)
|
||||
@@ -194,7 +197,9 @@
|
||||
builder (cond-> ^S3AsyncClientBuilder builder
|
||||
(some? endpoint)
|
||||
(.endpointOverride (URI. endpoint)))]
|
||||
(.build ^S3AsyncClientBuilder builder))]
|
||||
(.build ^S3AsyncClientBuilder builder))
|
||||
|
||||
]
|
||||
|
||||
(reify
|
||||
clojure.lang.IDeref
|
||||
@@ -221,7 +226,6 @@
|
||||
[id subscriber sem content]
|
||||
(px/thread
|
||||
{:name "penpot/s3/uploader"
|
||||
:virtual true
|
||||
:daemon true}
|
||||
(l/trace :hint "start upload thread"
|
||||
:object-id (str id)
|
||||
@@ -263,15 +267,15 @@
|
||||
(Optional/of (long (impl/get-size content))))
|
||||
|
||||
(^void subscribe [_ ^Subscriber subscriber]
|
||||
(let [sem (Semaphore. 0)
|
||||
thr (upload-thread id subscriber sem content)]
|
||||
(.onSubscribe subscriber
|
||||
(reify Subscription
|
||||
(cancel [_]
|
||||
(px/interrupt! thr)
|
||||
(.release sem 1))
|
||||
(request [_ n]
|
||||
(.release sem (int n)))))))))
|
||||
(let [sem (Semaphore. 0)
|
||||
thr (upload-thread id subscriber sem content)]
|
||||
(.onSubscribe subscriber
|
||||
(reify Subscription
|
||||
(cancel [_]
|
||||
(px/interrupt! thr)
|
||||
(.release sem 1))
|
||||
(request [_ n]
|
||||
(.release sem (int n)))))))))
|
||||
|
||||
|
||||
(defn- put-object
|
||||
@@ -309,7 +313,7 @@
|
||||
;; to the filesystem and then read with buffered inputstream; if
|
||||
;; not, read the contento into memory using bytearrays.
|
||||
(if (> ^long size (* 1024 1024 2))
|
||||
(let [path (tmp/tempfile :prefix "penpot.storage.s3." :min-age "6h")
|
||||
(let [path (tmp/tempfile :prefix "penpot.storage.s3.")
|
||||
rxf (AsyncResponseTransformer/toFile ^Path path)]
|
||||
(->> (.getObject ^S3AsyncClient client
|
||||
^GetObjectRequest gor
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user