🐛 Fix minor issues on event instumentation module

Hotfixes

.circleci/config.yml

@@ -36,17 +36,30 @@ jobs:
       - run:
           name: common lint
           working_directory: "./common"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
           name: frontend lint
           working_directory: "./frontend"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
+
+      - run:
+          name: frontend styles prettier
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run lint-scss
 
       - run:
           name: backend lint
           working_directory: "./backend"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       # run backend test
       - run:
@@ -68,21 +81,27 @@ jobs:
             node target/tests.js
 
           environment:
-            JAVA_HOME: /usr/lib/jvm/openjdk16
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+
+      # - run:
+      #     working_directory: "./common"
+      #     name: common tests (cljs)
+      #     command: |
+      #       yarn install
+      #       yarn run compile-test
+      #       node target/test.js
+      # 
+      #     environment:
+      #       PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
 
       - run:
           working_directory: "./common"
-          name: common tests
+          name: common tests (clj)
           command: |
-            yarn install
-            clojure -M:dev:shadow-cljs compile test
-            node target/tests.js
             clojure -X:dev:test
 
           environment:
-            JAVA_HOME: /usr/lib/jvm/openjdk16
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
 
       - save_cache:
          paths:

.clj-kondo/config.edn

@@ -3,13 +3,13 @@
   rumext.alpha/defc clojure.core/defn
   rumext.alpha/fnc clojure.core/fn
   app.common.data/export clojure.core/def
-  app.db/with-atomic clojure.core/with-open}
+  app.db/with-atomic clojure.core/with-open
+  app.common.logging/with-context clojure.core/do}
 
  :hooks
  {:analyze-call
   {app.common.data/export hooks.export/export
    potok.core/reify hooks.export/potok-reify
-   cljs.core/specify! hooks.export/clojure-specify
    app.util.services/defmethod hooks.export/service-defmethod
    }}
 
@@ -17,12 +17,16 @@
  {:exclude-files
   ["data_readers.clj"
    "app/util/perf.cljs"
+   "app/common/logging.cljc"
    "app/common/exceptions.cljc"]}
 
  :linters
  {:unsorted-required-namespaces
   {:level :warning}
 
+  :potok/reify-type
+  {:level :error}
+
   :unresolved-namespace
   {:level :warning
    :exclude [data_readers]}

.clj-kondo/hooks/export.clj

@@ -10,15 +10,34 @@
                   sname])]
     {:node result}))
 
+(def registry (atom {}))
+
 (defn potok-reify
-  [{:keys [:node]}]
+  [{:keys [:node :filename] :as params}]
   (let [[rnode rtype & other] (:children node)
-        result  (api/list-node
-                 (into [(api/token-node (symbol "deftype"))
-                        (api/token-node (gensym (name (:k rtype))))
-                        (api/vector-node [])]
-                       other))]
-    {:node result}))
+        rsym (symbol (str "event-type-" (name (:k rtype))))
+        reg  (get @registry filename #{})]
+    (when-not (:namespaced? rtype)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message "ptk/reify type should be namespaced"
+                           :type :potok/reify-type
+                           :row row
+                           :col col})))
+
+    (if (contains? reg rsym)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message (str "duplicate type: " (name (:k rtype)))
+                           :type :potok/reify-type
+                           :row row
+                           :col col}))
+      (swap! registry update filename (fnil conj #{}) rsym))
+
+    (let [result  (api/list-node
+                   (into [(api/token-node (symbol "deftype"))
+                          (api/token-node rsym)
+                          (api/vector-node [])]
+                         other))]
+      {:node result})))
 
 (defn clojure-specify
   [{:keys [:node]}]
@@ -32,18 +51,26 @@
 
 (defn service-defmethod
   [{:keys [:node]}]
-  (let [[rnode rtype & other] (:children node)
+  (let [[rnode rtype ?meta & other] (:children node)
         rsym    (gensym (name (:k rtype)))
         result  (api/list-node
                  [(api/token-node (symbol "do"))
                   (api/list-node
                    [(api/token-node (symbol "declare"))
                     (api/token-node rsym)])
+                  (if (= :map (:tag ?meta))
+                    (api/list-node
+                     [(api/token-node (symbol "reset-meta!"))
+                      (api/token-node rsym)
+                      ?meta])
+                    (api/list-node
+                     [(api/token-node (symbol "comment"))
+                      (api/token-node rsym)]))
                   (api/list-node
                    (into [(api/token-node (symbol "defmethod"))
                           (api/token-node rsym)
                           rtype]
-                         other))])]
+                         (cons ?meta other)))])]
+    ;; (prn "==============" rtype (into {} ?meta))
+    ;; (prn (api/sexpr result))
     {:node result}))
-
-

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,49 +8,48 @@ assignees: ''
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
-
 **To Reproduce**
+
 Steps to reproduce the behavior:
 1.  Go to '...'
 2.  Click on '....'
 3.  Scroll down to '....'
-4.  See error
 
 **Expected behavior**
+
 A clear and concise description of what you expected to happen.
 
+**Actual behavior**
+
+A clear and concise description of what happens instead; what the bug is.
+ 
 **Screenshots**
+
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
-
--   OS: (e.g. iOS)
--   Browser (e.g. chrome, safari)
--   Version (e.g. 22)
+- OS (e.g. iOS):
+- Browser & version (e.g. Chrome 89.0):
 
 **Smartphone (please complete the following information):**
-
--   Device: (e.g. iPhone6)
--   OS: (e.g. iOS8.1)
--   Browser (e.g. stock browser, safari)
--   Version (e.g. 22)
+- Device & model (e.g. iPhone 6):
+- OS & version (e.g. iOS 8.1):
+- Browser & version (e.g. stock browser 22):
 
 **Environment (please complete the following information):**
-Specify if using SAAS (https://design.penpot.app) or self-hosted instance.
+- Host (e.g. https://design.penpot.app, local instance):
 
-If self-hosted instance, add OS and runtime information to help explain your problem.
+*If self-hosted:*
+- OS Version (e.g. Ubuntu 16.04):
+- Docker / Docker-compose version (e.g. Docker version 18.03.0-ce, build 0520e24):
+- Image version (e.g. Alpine):
 
--   OS Version: (e.g. Ubuntu 16.04)
+Docker commands or docker-compose file (if possible and if proceed.x):
+```
 
-Also provide Docker commands or docker-compose file if possible and if proceed.x
-
--   Docker / Docker-compose Version: (e.g. Docker version 18.03.0-ce, build 0520e24)
--   Image (e.g. alpine)
-
-**Frontend Stack Trace (if self-hosted)**
+```
 
+Frontend Stack Trace:
 <details>
 
 ```
@@ -59,8 +58,7 @@ Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 </details>
 
-**Backend Stack Trace (if self-hosted)**
-
+Backend Stack Trace:
 <details>
 
 ```
@@ -69,5 +67,6 @@ Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 </details>
 
-**Additional context**
-Add any other context about the problem here.
+**Additional context:**
+
+Any other context about the problem.

.gitignore

@@ -1,41 +1,54 @@
-figwheel_server.log
-*jar
 *-init.clj
+*.jar
+*.penpot
+*.orig
+.calva
+.clj-kondo
+.cpcache
 .lein-deps-sum
 .lein-failures
-.lein-repl-history
 .lein-plugins/
-.repl
+.lein-repl-history
+.lsp
 .nrepl-port
-.cpcache
+.nyc_output
 .rebel_readline_history
-/vendor/**/target
-/cd.md
-node_modules
-/backend/target/
-/backend/resources/public/media
-/backend/resources/public/assets
+.repl
+/.clj-kondo/.cache
+/_dump
+/backend/-
 /backend/assets/
 /backend/dist/
 /backend/logs/
-/backend/-
-/telemetry/
-/frontend/npm-debug.log
-/frontend/target/
-/frontend/dist/
-/frontend/out/
-/frontend/.shadow-cljs
-/frontend/resources/public/*
-/frontend/resources/fonts/experiments
-/exporter/target
-/exporter/.shadow-cljs
-/docker/images/bundle*
-/common/.shadow-cljs
-/common/target
-/.clj-kondo/.cache
+/backend/resources/public/assets
+/backend/resources/public/media
+/backend/target/
 /bundle*
-/media
+/cd.md
+/clj-profiler/
+/common/.shadow-cljs
+/common/coverage
+/common/target
 /deploy
+/docker/images/bundle*
+/exporter/.shadow-cljs
+/exporter/target
+/frontend/.shadow-cljs
+/frontend/package-lock.json
+/frontend/cypress/videos/*/
+/frontend/cypress/fixtures/validuser.json
+/frontend/dist/
+/frontend/npm-debug.log
+/frontend/out/
+/frontend/resources/fonts/experiments
+/frontend/resources/public/*
+/frontend/target/
+/frontend/cypress/videos/*/
+/media
+/telemetry/
+/vendor/**/target
+/vendor/svgclean/bundle*.js
 /web
-/_dump
-/vendor/svgclean/bundle*.js
+clj-profiler/
+figwheel_server.log
+node_modules

CHANGES.md

@@ -1,14 +1,415 @@
-# CHANGELOG #
+# CHANGELOG
 
-## :rocket: Next
+## 1.12.2-beta
+
+### :bug: Bugs fixed
+
+- Fix issue with guides over shape handlers [Taiga #3032](https://tree.taiga.io/project/penpot/issue/3032)
+- Fix problem with shift+ctrl+click to select [#1671](https://github.com/penpot/penpot/issues/1671)
+- Fix ellipsis in long page names [Taiga #2962](https://tree.taiga.io/project/penpot/issue/2962)
+
+## 1.12.1-beta
+
+### :bug: Bugs fixed
+
+- Fix length of names in sidebar [Taiga #2962](https://tree.taiga.io/project/penpot/issue/2962)
+- Fix issues on loki integration
+
+
+## 1.12.0-beta
+
+### :boom: Breaking changes
 
 ### :sparkles: New features
 
+- Open feedback in a new window [Taiga #2901](https://tree.taiga.io/project/penpot/us/2901)
+- Improve usage of file menu [Taiga #2853](https://tree.taiga.io/project/penpot/us/2853)
+- Rotation to snap to 15º intervals with shift [Taiga #2437](https://tree.taiga.io/project/penpot/issue/2437)
+- Support border radius and stroke properties for images [Taiga #497](https://tree.taiga.io/project/penpot/us/497)
+- Disallow using same password as user email [Taiga #2454](https://tree.taiga.io/project/penpot/us/2454)
+- Add configurable nudge amount [Taiga #910](https://tree.taiga.io/project/penpot/us/910)
+- Add stroke properties for image shapes [Taiga #497](https://tree.taiga.io/project/penpot/us/497)
+- On user settings, hide the theme selector as long as we only have one theme [Taiga #2610](https://tree.taiga.io/project/penpot/us/2610)
+- Automatically open comments from dashboard notifications [Taiga #2605](https://tree.taiga.io/project/penpot/us/2605)
+- Enhance the behaviour of the artboards list on view mode [Taiga #2634](https://tree.taiga.io/project/penpot/us/2634)
+- Add recent used fonts in font selection widget [Taiga #1381](https://tree.taiga.io/project/penpot/us/1381)
+- Allow to align items relative to groups [Taiga #2533](https://tree.taiga.io/project/penpot/us/2533)
+- Scroll bars [Taiga #2550](https://tree.taiga.io/project/penpot/task/2550)
+- Add select layer option to context menu [Taiga #2474](https://tree.taiga.io/project/penpot/us/2474)
+- Guides [Taiga #290](https://tree.taiga.io/project/penpot/us/290)
+- Improve file menu by adding semantically groups [Github #1203](https://github.com/penpot/penpot/issues/1203)
+- Add update components in bulk option in context menu [Taiga #1975](https://tree.taiga.io/project/penpot/us/1975)
+- Create first E2E tests [Taiga #2608](https://tree.taiga.io/project/penpot/task/2608), [Taiga #2608](https://tree.taiga.io/project/penpot/task/2608)
+- Redesign of workspace toolbars [Taiga #2319](https://tree.taiga.io/project/penpot/us/2319)
+- Graphic Tablet usability improvements [Taiga #1913](https://tree.taiga.io/project/penpot/us/1913)
+- Improved mouse collision detection for groups and text shapes [Taiga #2452](https://tree.taiga.io/project/penpot/us/2452), [Taiga #2453](https://tree.taiga.io/project/penpot/us/2453)
+- Add support for alternative S3 storage providers and all aws regions [#1267](https://github.com/penpot/penpot/issues/1267)
+
 ### :bug: Bugs fixed
-### :arrow_up: Deps updates
-### :boom: Breaking changes
+
+- Fixed ungroup typography when editing it [Taiga #2391](https://tree.taiga.io/project/penpot/issue/2391) 
+- Fixed error when trying to post an empty comment [Taiga #2603](https://tree.taiga.io/project/penpot/issue/2603)
+- Fixed missing translation strings [Taiga #2786](https://tree.taiga.io/project/penpot/issue/2786)
+- Fixed color palette outside viewport [Taiga #2715](https://tree.taiga.io/project/penpot/issue/2715)
+- Fixed missing translate string [Taiga #2780](https://tree.taiga.io/project/penpot/issue/2780)
+- Fixed handoff shadow type text [Taiga #2717](https://tree.taiga.io/project/penpot/issue/2717)
+- Fixed components get "dirty" marker when moved [Taiga #2764](https://tree.taiga.io/project/penpot/issue/2764)
+- Fixed cannot align objects in a group that is not part of a frame [Taiga #2762](https://tree.taiga.io/project/penpot/issue/2762)
+- Fix problem with double click on exit path editing [Taiga #2906](https://tree.taiga.io/project/penpot/issue/2906)
+- Fixed alignment of layers with children [Taiga #2862](https://tree.taiga.io/project/penpot/issue/2862)
+
 ### :heart: Community contributions by (Thank you!)
 
+- Cleanup unused static images (by @rhcarvalho) [#1561](https://github.com/penpot/penpot/pull/1561)
+- Compress static images to save space (by @rhcarvalho) [#1562](https://github.com/penpot/penpot/pull/1562)
+
+## 1.11.2-beta
+
+### :bug: Bugs fixed
+
+- Fix issue on handling empty content on boolean shapes
+- Fix race condition issue on component renaming
+- Handle EOF errors on writting streamed response
+- Handle EOF errors on websocket send/ping methods
+- Disable parallel upload of file media on import (causes too much
+  contention on the rlimit subsistem that does not works as expected
+  on high load).
+
+### :sparkles: New features
+
+- Add health check endpoint on API
+- Increase default max connection pool size to 60
+- Reduce resource usage of the error reporter.
+
+## 1.11.1-beta
+
+### :bug: Bugs fixed
+
+- Fix issue related to default http host config value.
+- Fix issue on rendering frames on firefox.
+
+### :arrow_up: Deps updates
+
+- Update nodejs version to 16.13.1 on docker images.
+
+## 1.11.0-beta
+
+### :sparkles: New features
+
+- Add an option to hide artboards names on the viewport [Taiga #2034](https://tree.taiga.io/project/penpot/issue/2034)
+- Limit pasted object position to container boundaries [Taiga #2449](https://tree.taiga.io/project/penpot/us/2449)
+- Add new options for zoom widget in workspace and viewer mode [Taiga #896](https://tree.taiga.io/project/penpot/us/896)
+- Allow decimals on stroke width and positions [Taiga #2035](https://tree.taiga.io/project/penpot/issue/2035)
+- Ability to ignore background when exporting an artboard [Taiga #1395](https://tree.taiga.io/project/penpot/us/1395)
+- Show color hex or name on hover [Taiga #2413](https://tree.taiga.io/project/penpot/us/2413)
+- Add shortcut to create artboard from selected objects [Taiga #2412](https://tree.taiga.io/project/penpot/us/2412)
+- Add shortcut for opacity [Taiga #2442](https://tree.taiga.io/project/penpot/us/2442)
+- Setting fill automatically for new texts [Taiga #2441](https://tree.taiga.io/project/penpot/us/2441)
+- Add shortcut to move action [Github #1213](https://github.com/penpot/penpot/issues/1213)
+- Add alt as mod key to add stroke color from library menu [Taiga #2207](https://tree.taiga.io/project/penpot/us/2207)
+- Add detach in bulk option to context menu [Taiga #2210](https://tree.taiga.io/project/penpot/us/2210)
+- Add penpot look and feel to multiuser cursors [Taiga #1387](https://tree.taiga.io/project/penpot/us/1387)
+- Add actions to go to main component context menu option [Taiga #2053](https://tree.taiga.io/project/penpot/us/2053)
+- Add contrast between component select color and shape select color [Taiga #2121](https://tree.taiga.io/project/penpot/issue/2121)
+- Add animations in interactions [Taiga #2244](https://tree.taiga.io/project/penpot/us/2244)
+- Add performance improvements on .penpot file import process [Taiga #2497](https://tree.taiga.io/project/penpot/us/2497)
+- On team settings set color of members count to black [Taiga #2607](https://tree.taiga.io/project/penpot/us/2607)
+
+### :bug: Bugs fixed
+
+- Fix remove gradient if any when applying color from library [Taiga #2299](https://tree.taiga.io/project/penpot/issue/2299)
+- Fix Enter as key action to exit edit path [Taiga #2444](https://tree.taiga.io/project/penpot/issue/2444)
+- Fix add fill color from palette to groups and components [Taiga #2313](https://tree.taiga.io/project/penpot/issue/2313)
+- Fix default project name in all languages [Taiga #2280](https://tree.taiga.io/project/penpot/issue/2280)
+- Fix line-height and letter-spacing inputs to allow negative values [Taiga #2381](https://tree.taiga.io/project/penpot/issue/2381)
+- Fix typo in Handoff tooltip [Taiga #2428](https://tree.taiga.io/project/penpot/issue/2428)
+- Fix crash when pressing Shift+1 on empty file [#1435](https://github.com/penpot/penpot/issues/1435)
+- Fix masked group resize strange behavior [Taiga #2317](https://tree.taiga.io/project/penpot/issue/2317)
+- Fix problems when exporting all artboards [Taiga #2234](https://tree.taiga.io/project/penpot/issue/2234)
+- Fix problems with team management [#1353](https://github.com/penpot/penpot/issues/1353)
+- Fix problem when importing in shared libraries [#1362](https://github.com/penpot/penpot/issues/1362)
+- Fix problem with join nodes [#1422](https://github.com/penpot/penpot/issues/1422)
+- After team onboarding importing a file will import into the team drafts [Taiga #2408](https://tree.taiga.io/project/penpot/issue/2408)
+- Fix problem exporting shapes from handoff mode [Taiga #2386](https://tree.taiga.io/project/penpot/issue/2386)
+- Fix lock/hide elements in context menu when multiples shapes selected [Taiga #2340](https://tree.taiga.io/project/penpot/issue/2340)
+- Fix problem with booleans [Taiga #2356](https://tree.taiga.io/project/penpot/issue/2356)
+- Fix line-height/letter-spacing inputs behaviour [Taiga #2331](https://tree.taiga.io/project/penpot/issue/2331)
+- Fix dotted style in strokes [Taiga #2312](https://tree.taiga.io/project/penpot/issue/2312)
+- Fix problem when resizing texts inside groups [Taiga #2310](https://tree.taiga.io/project/penpot/issue/2310)
+- Fix problem with multiple exports [Taiga #2468](https://tree.taiga.io/project/penpot/issue/2468)
+- Allow import to continue from recoverable failures [#1412](https://github.com/penpot/penpot/issues/1412)
+- Improved behaviour on text options when not text is selected [Taiga #2390](https://tree.taiga.io/project/penpot/issue/2390)
+- Fix decimal numbers in export viewbox [Taiga #2290](https://tree.taiga.io/project/penpot/issue/2290)
+- Right click over artboard name to open its menu [Taiga #1679](https://tree.taiga.io/project/penpot/issue/1679)
+- Make the default session cookue use SameSite=Lax instead of Strict (causes some issues in latest versions of Chrome)
+- Fix "open in new tab" on dashboard [Taiga #2235](https://tree.taiga.io/project/penpot/issue/2355)
+- Changing pages while comments activated will not close the panel [#1350](https://github.com/penpot/penpot/issues/1350)
+- Fix navigate comments in right sidebar [Taiga #2163](https://tree.taiga.io/project/penpot/issue/2163)
+- Fix keep name of component equal to the shape name [Taiga #2341](https://tree.taiga.io/project/penpot/issue/2341)
+- Fix lossing changes when changing selection and an input was already changed [Taiga #2329](https://tree.taiga.io/project/penpot/issue/2329), [Taiga #2330](https://tree.taiga.io/project/penpot/issue/2330)
+- Fix blur input field when click on viewport [Taiga #2164](https://tree.taiga.io/project/penpot/issue/2164)
+- Fix default page id in workspace [Taiga #2205](https://tree.taiga.io/project/penpot/issue/2205)
+- Fix problem when importing a file with grids [Taiga #2314](https://tree.taiga.io/project/penpot/issue/2314)
+- Fix problem with imported svgs with filters [Taiga #2478](https://tree.taiga.io/project/penpot/issue/2478)
+- Fix issues when updating selrect in paths [Taiga #2366](https://tree.taiga.io/project/penpot/issue/2366)
+- Fix scroll jumps in handoff mode [Taiga #2383](https://tree.taiga.io/project/penpot/issue/2383)
+- Fix handoff text with opacity [Taiga #2384](https://tree.taiga.io/project/penpot/issue/2384)
+- Restored rules color [Taiga #2460](https://tree.taiga.io/project/penpot/issue/2460)
+- Fix thumbnail not taking frame blending mode [Taiga #2301](https://tree.taiga.io/project/penpot/issue/2301)
+- Fix import/export with SVG edge cases [Taiga #2389](https://tree.taiga.io/project/penpot/issue/2389)
+- Avoid modifying component when moving into a group [Taiga #2534](https://tree.taiga.io/project/penpot/issue/2534)
+- Show correctly group types label in handoff [Taiga #2482](https://tree.taiga.io/project/penpot/issue/2482)
+- Display view mode buttons always centered in viewer [#Taiga 2466](https://tree.taiga.io/project/penpot/issue/2466)
+- Fix default profile image generation issue [Taiga #2601](https://tree.taiga.io/project/penpot/issue/2601)
+- Fix edit blur attributes for multiselection [Taiga #2625](https://tree.taiga.io/project/penpot/issue/2625)
+- Fix auto hide header in viewer full screen [Taiga #2632](https://tree.taiga.io/project/penpot/issue/2632)
+- Fix zoom in/out after fit or fill [Taiga #2630](https://tree.taiga.io/project/penpot/issue/2630)
+- Normalize zoom levels in workspace and viewer [Taiga #2631](https://tree.taiga.io/project/penpot/issue/2631)
+- Avoid empty names in projects, files and pages [Taiga #2594](https://tree.taiga.io/project/penpot/issue/2594)
+- Fix "move to" menu when duplicated team or project names [Taiga #2655](https://tree.taiga.io/project/penpot/issue/2655)
+- Fix ungroup a component leaves an asterisk in layers [Taiga #2694](https://tree.taiga.io/project/penpot/issue/2694)
+
+### :arrow_up: Deps updates
+
+- Update devenv docker image dependencies
+
+### :heart: Community contributions by (Thank you!)
+
+- Spelling fixes (by @jsoref) [#1340](https://github.com/penpot/penpot/pull/1340)
+- Explain folders in components (by @candideu) [Penpot-docs #42](https://github.com/penpot/penpot-docs/pull/42)
+- Readability improvements of user guide (by @PaulSchulz) [Penpot-docs #50](https://github.com/penpot/penpot-docs/pull/50)
+
+## 1.10.4-beta
+
+### :sparkles: Enhacements
+
+- Allow parametrice file snapshoting interval
+
+### :bug: Bugs fixed
+
+- Fix issue on :mov-object change impl
+- Minor fix on how file changes log is persisted
+- Fix many issues on error reporting
+
+## 1.10.3-beta
+
+### :sparkles: Enhacements
+
+- Make all logging asynchronous, this avoid some overhead on jetty threads at cost of logging latency.
+- Increase default session time to 15 days.
+
+### :bug: Bugs fixed
+
+- Fix unexpected exception on saving pages with default grids [#2409](https://tree.taiga.io/project/penpot/issue/2409)
+- Fix react warnings on setting size 1 on row and column grids.
+- Fix minor issues on ZMQ logging listener (used in error reporting service)
+- Remove "ALPHA" from the code.
+- Fix value and nil handling on numeric-input component. This fixes many issues related to typography, components, etc. renaming.
+- Fix NPE on email complains processing.
+- Fix white page after leaving a team.
+- Fix missing leave team button outside members page.
+
+### :arrow_up: Deps updates
+
+- Update log4j2 dependency.
+
+## 1.10.2-beta
+
+### :bug: Bugs fixed
+
+- Fix corner case issues with media file uploads.
+- Fix issue with default page grids validation.
+- Fix issue related to some raceconditions on workspace navigation events.
+
+### :arrow_up: Deps updates
+
+- Update log4j2 dependency.
+
+## 1.10.1-beta
+
+### :bug: Bugs fixed
+
+- Fix problems with team management [#1353](https://github.com/penpot/penpot/issues/1353)
+
+## 1.10.0-beta
+
+### :boom: Breaking changes
+
+- The initial project / data mechanism (not documented) has been
+  disabled. Is the mechanism used for creating initial project on user
+  signup. With the new onboarding approach, this subsystem is no
+  longer needed and is disabled.
+
+### :sparkles: New features
+
+- Allow ungroup groups in bulk [Taiga #2211](https://tree.taiga.io/project/penpot/us/2211)
+- Enhance corner radius behavior [Taiga #2190](https://tree.taiga.io/project/penpot/issue/2190)
+- Allow preserve scroll position in interactions [Taiga #2250](https://tree.taiga.io/project/penpot/us/2250)
+- Add new onboarding modals.
+
+### :bug: Bugs fixed
+
+- Fix problem with exporting before the document is saved [Taiga #2189](https://tree.taiga.io/project/penpot/issue/2189)
+- Fix undo stacking when changing color from color-picker [Taiga #2191](https://tree.taiga.io/project/penpot/issue/2191)
+- Fix pages dropdown in viewer [Taiga #2087](https://tree.taiga.io/project/penpot/issue/2087)
+- Fix problem when exporting texts with gradients or opacity [Taiga #2200](https://tree.taiga.io/project/penpot/issue/2200)
+- Fix problem with view mode comments [Taiga #2226](https://tree.taiga.io/project/penpot/issue/2226)
+- Disallow to create a component when already has one [Taiga #2237](https://tree.taiga.io/project/penpot/issue/2237)
+- Add ellipsis in long labels for input fields [Taiga #2224](https://tree.taiga.io/project/penpot/issue/2224)
+- Fix problem with text rendering on export [Taiga #2223](https://tree.taiga.io/project/penpot/issue/2223)
+- Fix problem when flattening booleans losing styles [Taiga #2217](https://tree.taiga.io/project/penpot/issue/2217)
+- Add shortcuts to boolean icons popups [Taiga #2220](https://tree.taiga.io/project/penpot/issue/2220)
+- Fix a worker error when transforming a rectangle into path
+- Fix max/min values for opacity fields [Taiga #2183](https://tree.taiga.io/project/penpot/issue/2183)
+- Fix viewer comment position when zoom applied [Taiga #2240](https://tree.taiga.io/project/penpot/issue/2240)
+- Remove change style on hover for options [Taiga #2172](https://tree.taiga.io/project/penpot/issue/2172)
+- Fix problem in viewer with dropdowns when comments active [#1303](https://github.com/penpot/penpot/issues/1303)
+- Add placeholder to create shareable link
+- Fix project files count not refreshing correctly after import [Taiga #2216](https://tree.taiga.io/project/penpot/issue/2216)
+- Remove button after import process finish [Taiga #2215](https://tree.taiga.io/project/penpot/issue/2215)
+- Fix problem with styles in the viewer [Taiga #2467](https://tree.taiga.io/project/penpot/issue/2467)
+- Fix default state in viewer [Taiga #2465](https://tree.taiga.io/project/penpot/issue/2465)
+- Fix division by zero in bool operation [Taiga #2349](https://tree.taiga.io/project/penpot/issue/2349)
+
+### :heart: Community contributions by (Thank you!)
+
+- To the translation community for the hard work on making penpot
+  available on so many languages.
+- Guide to integrate with Azure Directory (by @skrzyneckik) [Penpot-docs #33](https://github.com/penpot/penpot-docs/pull/33)
+- Improve libraries section readability (by @PaulSchulz) [Penpot-docs #39](https://github.com/penpot/penpot-docs/pull/39)
+
+## 1.9.0-alpha
+
+### :boom: Breaking changes
+
+- Some stroke-caps can change behaviour.
+- Text display bug fix could potentially make some texts jump a line.
+
+### :sparkles: New features
+
+- Add boolean shapes: intersections, unions, difference and exclusions[Taiga #748](https://tree.taiga.io/project/penpot/us/748)
+- Add advanced prototyping [Taiga #244](https://tree.taiga.io/project/penpot/us/244)
+- Add multiple flows [Taiga #2091](https://tree.taiga.io/project/penpot/us/2091)
+- Change order of the teams menu so it's in the joined time order.
+
+### :bug: Bugs fixed
+
+- Enhance duplicating prototype connections behaviour [Taiga #2093](https://tree.taiga.io/project/penpot/us/2093)
+- Ignore constraints in horizontal or vertical flip [Taiga #2038](https://tree.taiga.io/project/penpot/issue/2038)
+- Fix color and typographies refs lost when duplicated file [Taiga #2165](https://tree.taiga.io/project/penpot/issue/2165)
+- Fix problem with overflow dropdown on stroke-cap [#1216](https://github.com/penpot/penpot/issues/1216)
+- Fix menu context for single element nested in components [#1186](https://github.com/penpot/penpot/issues/1186)
+- Fix error screen when operations over comments fail [#1219](https://github.com/penpot/penpot/issues/1219)
+- Fix undo problem when changing typography/color from library [#1230](https://github.com/penpot/penpot/issues/1230)
+- Fix problem with text margin while rendering [#1231](https://github.com/penpot/penpot/issues/1231)
+- Fix problem with masked texts on exporting [Taiga #2116](https://tree.taiga.io/project/penpot/issue/2116)
+- Fix text editor enter behaviour with centered texts [Taiga #2126](https://tree.taiga.io/project/penpot/issue/2126)
+- Fix residual stroke on imported svg [Taiga #2125](https://tree.taiga.io/project/penpot/issue/2125)
+- Add links for terms of service and privacy policy in register checkbox [Taiga #2020](https://tree.taiga.io/project/penpot/issue/2020)
+- Allow three character hex and web colors in color picker hex input [#1184](https://github.com/penpot/penpot/issues/1184)
+- Allow lowercase search for fonts [#1180](https://github.com/penpot/penpot/issues/1180)
+- Fix group renaming problem [Taiga #1969](https://tree.taiga.io/project/penpot/issue/1969)
+- Fix export group with shadows on children [Taiga #2036](https://tree.taiga.io/project/penpot/issue/2036)
+- Fix zoom context menu in viewer [Taiga #2041](https://tree.taiga.io/project/penpot/issue/2041)
+- Fix stroke caps adjustments in relation with stroke size [Taiga #2123](https://tree.taiga.io/project/penpot/issue/2123)
+- Fix problem duplicating paths [Taiga #2147](https://tree.taiga.io/project/penpot/issue/2147)
+- Fix problem inheriting attributes from SVG root when importing [Taiga #2124](https://tree.taiga.io/project/penpot/issue/2124)
+- Fix problem with lines and inside/outside stroke [Taiga #2146](https://tree.taiga.io/project/penpot/issue/2146)
+- Add stroke width in selection calculation [Taiga #2146](https://tree.taiga.io/project/penpot/issue/2146)
+- Fix shift+wheel to horizontal scrolling in MacOS [#1217](https://github.com/penpot/penpot/issues/1217)
+- Fix path stroke is not working properly with high thickness [Taiga #2154](https://tree.taiga.io/project/penpot/issue/2154)
+- Fix bug with transformation operations [Taiga #2155](https://tree.taiga.io/project/penpot/issue/2155)
+- Fix bug in firefox when a text box is inside a mask [Taiga #2152](https://tree.taiga.io/project/penpot/issue/2152)
+- Fix problem with stroke inside/outside [Taiga #2186](https://tree.taiga.io/project/penpot/issue/2186)
+- Fix masks export area [Taiga #2189](https://tree.taiga.io/project/penpot/issue/2189)
+- Fix paste in place in artboards [Taiga #2188](https://tree.taiga.io/project/penpot/issue/2188)
+- Fix font size input stuck on selection change [Taiga #2184](https://tree.taiga.io/project/penpot/issue/2184)
+- Fix stroke cut on shapes export [Taiga #2171](https://tree.taiga.io/project/penpot/issue/2171)
+- Fix no color when boolean with an SVG [Taiga #2193](https://tree.taiga.io/project/penpot/issue/2193)
+- Fix unlink color styles at strokes [Taiga #2206](https://tree.taiga.io/project/penpot/issue/2206)
+
+### :arrow_up: Deps updates
+
+### :heart: Community contributions by (Thank you!)
+
+- To the translation community for the hard work on making penpot
+  available on so many languages.
+
+## 1.8.4-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem importing components [Taiga #2151](https://tree.taiga.io/project/penpot/issue/2151)
+
+## 1.8.3-alpha
+
+### :sparkles: New features
+
+- Adds progress report to importing process.
+
+## 1.8.2-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with masking images in viewer [#1238](https://github.com/penpot/penpot/issues/1238)
+
+## 1.8.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix project renaming issue (and some other related to the same underlying bug)
+- Fix internal exception on audit log persistence layer.
+- Set proper environment variable on docker images for chrome executable.
+- Fix internal metrics on websocket connections.
+
+## 1.8.0-alpha
+
+### :boom: Breaking changes
+
+- This release includes a new approach for handling share links, and
+  this feature is incompatible with the previous one. This means that
+  all the public share links generated previously will stop working.
+
+### :sparkles: New features
+
+- Add tooltips to color picker tabs [Taiga #1814](https://tree.taiga.io/project/penpot/us/1814)
+- Add styling to the end point of any open paths [Taiga #1107](https://tree.taiga.io/project/penpot/us/1107)
+- Allow to zoom with ctrl + middle button [Taiga #1428](https://tree.taiga.io/project/penpot/us/1428)
+- Auto placement of duplicated objects [Taiga #1386](https://tree.taiga.io/project/penpot/us/1386)
+- Enable penpot SVG metadata only when exporting complete files [Taiga #1914](https://tree.taiga.io/project/penpot/us/1914?milestone=295883)
+- Export to PDF all artboards of one page [Taiga #1895](https://tree.taiga.io/project/penpot/us/1895)
+- Go to a undo step clicking on a history element of the list [Taiga #1374](https://tree.taiga.io/project/penpot/us/1374)
+- Increment font size by 10 with shift+arrows [1047](https://github.com/penpot/penpot/issues/1047)
+- New shortcut to detach components Ctrl+Shift+K [Taiga #1799](https://tree.taiga.io/project/penpot/us/1799)
+- Set email inputs to type "email", to aid keyboard entry [Taiga #1921](https://tree.taiga.io/project/penpot/issue/1921)
+- Use shift+move to move element orthogonally [#823](https://github.com/penpot/penpot/issues/823)
+- Use space + mouse drag to pan, instead of only space [Taiga #1800](https://tree.taiga.io/project/penpot/us/1800)
+- Allow navigate through pages on the viewer [Taiga #1550](https://tree.taiga.io/project/penpot/us/1550)
+- Allow create share links with specific pages [Taiga #1844](https://tree.taiga.io/project/penpot/us/1844)
+
+### :bug: Bugs fixed
+
+- Prevent adding numeric suffix to layer names when not needed [Taiga #1929](https://tree.taiga.io/project/penpot/us/1929)
+- Prevent deleting or moving the drafts project [Taiga #1935](https://tree.taiga.io/project/penpot/issue/1935)
+- Fix problem with zoom and selection [Taiga #1919](https://tree.taiga.io/project/penpot/issue/1919)
+- Fix problem with borders on shape export [#1092](https://github.com/penpot/penpot/issues/1092)
+- Fix thumbnail cropping issue [Taiga #1964](https://tree.taiga.io/project/penpot/issue/1964)
+- Fix repeated fetch on file selection [Taiga #1933](https://tree.taiga.io/project/penpot/issue/1933)
+- Fix rename typography on text options [Taiga #1963](https://tree.taiga.io/project/penpot/issue/1963)
+- Fix problems with order in groups [Taiga #1960](https://tree.taiga.io/project/penpot/issue/1960)
+- Fix SVG components preview [#1134](https://github.com/penpot/penpot/issues/1134)
+- Fix group renaming problem [Taiga #1969](https://tree.taiga.io/project/penpot/issue/1969)
+- Fix problem with import broken images links [#1197](https://github.com/penpot/penpot/issues/1197)
+- Fix problem while moving imported SVG's [#1199](https://github.com/penpot/penpot/issues/1199)
+
+### :arrow_up: Deps updates
+
+### :boom: Breaking changes
+
+### :heart: Community contributions by (Thank you!)
+
+- eduayme [#1129](https://github.com/penpot/penpot/pull/1129)
+
 ## 1.7.4-alpha
 
 ### :bug: Bugs fixed
@@ -16,14 +417,12 @@
 - Fix demo user creation (self-hosted only)
 - Add better ldap response validation and reporting (self-hosted only)
 
-
 ## 1.7.3-alpha
 
 ### :bug: Bugs fixed
 
 - Fix font uploading issue on Windows.
 
-
 ## 1.7.2-alpha
 
 ### :sparkles: New features
@@ -32,8 +431,8 @@
 
 ### :bug: Bugs fixed
 
-- Add scroll bar to Teams menu [Taiga #1894](https://tree.taiga.io/project/penpot/issue/1894).
-- Fix repeated names when duplicating artboards or groups [Taiga #1892](https://tree.taiga.io/project/penpot/issue/1892).
+- Add scroll bar to Teams menu [Taiga #1894](https://tree.taiga.io/project/penpot/issue/1894)
+- Fix repeated names when duplicating artboards or groups [Taiga #1892](https://tree.taiga.io/project/penpot/issue/1892)
 - Fix properly messages lifecycle on navigate.
 - Fix handling repeated names on duplicate object trees.
 - Fix group naming on group creation.
@@ -43,12 +442,10 @@
 
 - Update frontend build tooling.
 
-### :boom: Breaking changes
 ### :heart: Community contributions by (Thank you!)
 
 - soultipsy [#1100](https://github.com/penpot/penpot/pull/1100)
 
-
 ## 1.7.1-alpha
 
 ### :bug: Bugs fixed
@@ -58,19 +455,18 @@
 - Fix issue on undo page deletion.
 - Fix some issues related to constraints.
 
-
 ## 1.7.0-alpha
 
 ### :sparkles: New features
 
-- Allow nested asset groups [Taiga #1716](https://tree.taiga.io/project/penpot/us/1716).
-- Allow to ungroup assets [Taiga #1719](https://tree.taiga.io/project/penpot/us/1719).
-- Allow to rename assets groups [Taiga #1721](https://tree.taiga.io/project/penpot/us/1721).
-- Component constraints (left, right, left and right, center, scale...) [Taiga #1125](https://tree.taiga.io/project/penpot/us/1125).
-- Export elements to PDF [Taiga #519](https://tree.taiga.io/project/penpot/us/519).
-- Memorize collapse state of assets in panel [Taiga #1718](https://tree.taiga.io/project/penpot/us/1718).
-- Headers button sets and menus review [Taiga #1663](https://tree.taiga.io/project/penpot/us/1663).
-- Preserve components if possible, when pasted into a different file [Taiga #1063](https://tree.taiga.io/project/penpot/issue/1063).
+- Allow nested asset groups [Taiga #1716](https://tree.taiga.io/project/penpot/us/1716)
+- Allow to ungroup assets [Taiga #1719](https://tree.taiga.io/project/penpot/us/1719)
+- Allow to rename assets groups [Taiga #1721](https://tree.taiga.io/project/penpot/us/1721)
+- Component constraints (left, right, left and right, center, scale...) [Taiga #1125](https://tree.taiga.io/project/penpot/us/1125)
+- Export elements to PDF [Taiga #519](https://tree.taiga.io/project/penpot/us/519)
+- Memorize collapse state of assets in panel [Taiga #1718](https://tree.taiga.io/project/penpot/us/1718)
+- Headers button sets and menus review [Taiga #1663](https://tree.taiga.io/project/penpot/us/1663)
+- Preserve components if possible, when pasted into a different file [Taiga #1063](https://tree.taiga.io/project/penpot/issue/1063)
 - Add the ability to offload file data to a cheaper storage when file becomes inactive.
 - Import/Export Penpot files from dashboard.
 - Double click won't make a shape a path until you change a node [Taiga #1796](https://tree.taiga.io/project/penpot/us/1796)
@@ -79,25 +475,20 @@
 ### :bug: Bugs fixed
 
 - Process numeric input changes only if the value actually changed.
-- Remove unnecesary redirect from history when user goes to workspace from dashboard [Taiga #1820](https://tree.taiga.io/project/penpot/issue/1820).
-- Detach shapes from deleted assets [Taiga #1850](https://tree.taiga.io/project/penpot/issue/1850).
-- Fix tooltip position on view application [Taiga #1819](https://tree.taiga.io/project/penpot/issue/1819).
-- Fix dashboard navigation on moving file to other team [Taiga #1817](https://tree.taiga.io/project/penpot/issue/1817).
-- Fix workspace header presence styles and invalid link [Taiga #1813](https://tree.taiga.io/project/penpot/issue/1813).
-- Fix color-input wrong behavior (on workspace page color) [Taiga #1795](https://tree.taiga.io/project/penpot/issue/1795).
-- Fix file contextual menu in shared libraries at dashboard [Taiga #1865](https://tree.taiga.io/project/penpot/issue/1865).
+- Remove unnecessary redirect from history when user goes to workspace from dashboard [Taiga #1820](https://tree.taiga.io/project/penpot/issue/1820)
+- Detach shapes from deleted assets [Taiga #1850](https://tree.taiga.io/project/penpot/issue/1850)
+- Fix tooltip position on view application [Taiga #1819](https://tree.taiga.io/project/penpot/issue/1819)
+- Fix dashboard navigation on moving file to other team [Taiga #1817](https://tree.taiga.io/project/penpot/issue/1817)
+- Fix workspace header presence styles and invalid link [Taiga #1813](https://tree.taiga.io/project/penpot/issue/1813)
+- Fix color-input wrong behavior (on workspace page color) [Taiga #1795](https://tree.taiga.io/project/penpot/issue/1795)
+- Fix file contextual menu in shared libraries at dashboard [Taiga #1865](https://tree.taiga.io/project/penpot/issue/1865)
 - Fix problem with color picker and fonts [#1049](https://github.com/penpot/penpot/issues/1049)
 - Fix negative values in blur [Taiga #1815](https://tree.taiga.io/project/penpot/issue/1815)
 - Fix problem when editing color in group [Taiga #1816](https://tree.taiga.io/project/penpot/issue/1816)
 - Fix resize/rotate with mouse buttons different than left [#1060](https://github.com/penpot/penpot/issues/1060)
-- Fix header partialy visible on fullscreen viewer mode [Taiga #1875](https://tree.taiga.io/project/penpot/issue/1875)
+- Fix header partially visible on fullscreen viewer mode [Taiga #1875](https://tree.taiga.io/project/penpot/issue/1875)
 - Fix dynamic alignment enabled with hidden objects [#1063](https://github.com/penpot/penpot/issues/1063)
 
-
-### :arrow_up: Deps updates
-### :boom: Breaking changes
-### :heart: Community contributions by (Thank you!)
-
 ## 1.6.5-alpha
 
 ### :bug: Bugs fixed
@@ -108,8 +499,8 @@
 
 ### :sparkles: Minor improvements
 
--  Decrease default bulk buffers on storage tasks.
--  Reduce file_change preserve interval to 24h.
+- Decrease default bulk buffers on storage tasks.
+- Reduce file_change preserve interval to 24h.
 
 ### :bug: Bugs fixed
 
@@ -122,7 +513,6 @@
 - Properly handle nil values on `update-shapes` function.
 - Replace frame term usage by artboard on viewer app.
 
-
 ## 1.6.3-alpha
 
 ### :bug: Bugs fixed
@@ -149,42 +539,39 @@
 - Minor fix on previous commit.
 - Minor improvements on svg uploading on libraries.
 
-
 ## 1.6.1-alpha
 
 ### :bug: Bugs fixed
 
 - Add safety check on reg-objects change impl.
-- Fix custom fonts embbedding issue.
+- Fix custom fonts embedding issue.
 - Fix dashboard ordering issue.
 - Fix problem when creating a component with empty data.
 - Fix problem with moving shapes into frames.
 - Fix problems with mov-objects.
-- Fix unexpected excetion related to rounding integers.
+- Fix unexpected exception related to rounding integers.
 - Fix wrong type usage on libraries changes.
 - Improve editor lifecycle management.
 - Make the navigation async by default.
 
-
 ## 1.6.0-alpha
 
 ### :sparkles: New features
 
-- Add improved workspace font selector [Taiga US #292](https://tree.taiga.io/project/penpot/us/292).
+- Add improved workspace font selector [Taiga US #292](https://tree.taiga.io/project/penpot/us/292)
 - Add option to interactively scale text [Taiga #1527](https://tree.taiga.io/project/penpot/us/1527)
 - Add performance improvements on dashboard data loading.
 - Add performance improvements to indexes handling on workspace.
-- Add the ability to upload/use custom fonts (and automatically generate all needed webfonts) [Taiga US #292](https://tree.taiga.io/project/penpot/us/292).
+- Add the ability to upload/use custom fonts (and automatically generate all needed webfonts) [Taiga US #292](https://tree.taiga.io/project/penpot/us/292)
 - Transform shapes to path on double click
 - Translate automatic names of new files and projects.
-- Use shift instead of ctrl/cmd to keep aspect ratio [Taiga 1697](https://tree.taiga.io/project/penpot/issue/1697).
-- New translations: Portuguese (Brazil) and Romanias. 
-
+- Use shift instead of ctrl/cmd to keep aspect ratio [Taiga 1697](https://tree.taiga.io/project/penpot/issue/1697)
+- New translations: Portuguese (Brazil) and Romanias.
 
 ### :bug: Bugs fixed
 
-- Remove interactions when the destination artboard is deleted [Taiga #1656](https://tree.taiga.io/project/penpot/issue/1656).
-- Fix problem with fonts that ends with numbers [#940](https://github.com/penpot/penpot/issues/940).
+- Remove interactions when the destination artboard is deleted [Taiga #1656](https://tree.taiga.io/project/penpot/issue/1656)
+- Fix problem with fonts that ends with numbers [#940](https://github.com/penpot/penpot/issues/940)
 - Fix problem with imported SVG on editing paths [#971](https://github.com/penpot/penpot/issues/971)
 - Fix problem with color picker positioning
 - Fix order on color palette [#961](https://github.com/penpot/penpot/issues/961)
@@ -198,7 +585,6 @@
 - Update exporter dependencies (puppeteer), that fixes some unexpected exceptions.
 - Update string manipulation library.
 
-
 ### :boom: Breaking changes
 
 - The OIDC setting `PENPOT_OIDC_SCOPES` has changed the default semantics. Before this
@@ -209,7 +595,6 @@
 
 - Translations: Portuguese (Brazil) and Romanias.
 
-
 ## 1.5.4-alpha
 
 ### :bug: Bugs fixed
@@ -217,7 +602,6 @@
 - Fix issues on group rendering.
 - Fix problem with text editing auto-height [Taiga #1683](https://tree.taiga.io/project/penpot/issue/1683)
 
-
 ## 1.5.3-alpha
 
 ### :bug: Bugs fixed
@@ -241,7 +625,6 @@
 - Increase default team invitation token expiration to 48h.
 - Fix wrong error message when an expired token is used.
 
-
 ## 1.5.0-alpha
 
 ### :sparkles: New features
@@ -287,7 +670,6 @@
 - madmath03 (by [Monogramm](https://github.com/Monogramm)) [#807](https://github.com/penpot/penpot/pull/807)
 - zzkt [#814](https://github.com/penpot/penpot/pull/814)
 
-
 ## 1.4.1-alpha
 
 ### :bug: Bugs fixed
@@ -299,7 +681,6 @@
 - Fix incorrect state management of user lang selection.
 - Fix email validation usability issue on team invitation lightbox.
 
-
 ## 1.4.0-alpha
 
 ### :sparkles: New features
@@ -308,7 +689,7 @@
 - Add http caching layer on top of Query RPC.
 - Add layer opacity and blend mode to shapes [Taiga #937](https://tree.taiga.io/project/penpot/us/937)
 - Add more chinese translations [#726](https://github.com/penpot/penpot/pull/726)
-- Add native support for text-direction (RTL, LTR & auto).
+- Add native support for text-direction (RTL, LTR & auto)
 - Add several enhancements in shape selection [Taiga #1195](https://tree.taiga.io/project/penpot/us/1195)
 - Add thumbnail in memory caching mechanism.
 - Add turkish translation strings [#759](https://github.com/penpot/penpot/pull/759), [#794](https://github.com/penpot/penpot/pull/794)
@@ -316,13 +697,12 @@
 - Hide viewer navbar on fullscreen [Taiga 1375](https://tree.taiga.io/project/penpot/us/1375)
 - Import SVG will create Penpot's shapes [Taiga #1006](https://tree.taiga.io/project/penpot/us/1066)
 - Improve french translations [#731](https://github.com/penpot/penpot/pull/731)
-- Reimplement workspace presence (remove database state).
+- Reimplement workspace presence (remove database state)
 - Remember last visited team when you re-enter the application [Taiga #1376](https://tree.taiga.io/project/penpot/us/1376)
 - Rename artboard with double click on the title [Taiga #1392](https://tree.taiga.io/project/penpot/us/1392)
 - Replace Slate-Editor with DraftJS [Taiga #1346](https://tree.taiga.io/project/penpot/us/1346)
 - Set proper page title [Taiga #1377](https://tree.taiga.io/project/penpot/us/1377)
 
-
 ### :bug: Bugs fixed
 
 - Disable buttons in view mode for users without permissions [Taiga #1328](https://tree.taiga.io/project/penpot/issue/1328)
@@ -363,15 +743,13 @@
 
 - The LDAP configuration variables interpolation starts using `:`
   (example `:username`) instead of `$`. The main reason is avoid
-  unnecesary conflict with bash interpolation.
-
+  unnecessary conflict with bash interpolation.
 
 ### :arrow_up: Deps updates
 
 - Update backend to JDK16.
 - Update exporter nodejs to v14.16.0
 
-
 ### :heart: Community contributions by (Thank you!)
 
 - iblueer [#726](https://github.com/penpot/penpot/pull/726)
@@ -379,27 +757,25 @@
 - girafic [#748](https://github.com/penpot/penpot/pull/748)
 - mbrksntrk [#794](https://github.com/penpot/penpot/pull/794)
 
-
 ## 1.3.0-alpha
 
 ### :sparkles: New features
 
 - Add emailcatcher and ldap test containers to devenv. [#506](https://github.com/penpot/penpot/pull/506)
 - Add major refactor of internal pubsub/redis code; improves scalability and performance [#640](https://github.com/penpot/penpot/pull/640)
-- Add more chinese transtions [#687](https://github.com/penpot/penpot/pull/687)
+- Add more chinese translations [#687](https://github.com/penpot/penpot/pull/687)
 - Add more presets for artboard [#654](https://github.com/penpot/penpot/pull/654)
 - Add optional loki integration [#645](https://github.com/penpot/penpot/pull/645)
 - Add proper http session lifecycle handling.
 - Allow to set border radius of each rect corner individually
 - Bounce & Complaint handling [#635](https://github.com/penpot/penpot/pull/635)
 - Disable groups interactions when holding "Ctrl" key (deep selection)
-- New action in context menu to "edit" some shapes (binded to key "Enter")
-
+- New action in context menu to "edit" some shapes (bound to key "Enter")
 
 ### :bug: Bugs fixed
 
 - Add more improvements to french translation strings [#591](https://github.com/penpot/penpot/pull/591)
-- Add some missing database indexes (mainly improves performance on large databases on file-update rpc method, and some background tasks).
+- Add some missing database indexes (mainly improves performance on large databases on file-update rpc method, and some background tasks)
 - Disables filters in masking elements (issue with Firefox rendering)
 - Drawing tool will have priority over resize/rotate handlers [Taiga #1225](https://tree.taiga.io/project/penpot/issue/1225)
 - Fix broken bounding box on editing paths [Taiga #1254](https://tree.taiga.io/project/penpot/issue/1254)
@@ -413,16 +789,14 @@
 - Have language change notification written in the new language [Taiga #1205](https://tree.taiga.io/project/penpot/issue/1205)
 - Hide register screen when registration is disabled [#598](https://github.com/penpot/penpot/issues/598)
 - Properly handle errors on github, gitlab and ldap auth backends.
-- Properly mark profile auth backend (on first register/ auth with 3rd party auth provider).
+- Properly mark profile auth backend (on first register/ auth with 3rd party auth provider)
 - Refactor LDAP auth backend.
 
-
 ### :heart: Community contributions by (Thank you!)
 
 - girafic [#538](https://github.com/penpot/penpot/pull/654)
 - arkhi [#591](https://github.com/penpot/penpot/pull/591)
 
-
 ## 1.2.0-alpha
 
 ### :sparkles: New features
@@ -437,7 +811,6 @@
 - Show a pixel grid when zoom greater than 800% [#519](https://github.com/penpot/penpot/discussions/519)
 - Fix behavior of select all command when there are objects outside frames [Taiga #1209](https://tree.taiga.io/project/penpot/issue/1209)
 
-
 ### :bug: Bugs fixed
 
 - Fix 404 when access shared link [#615](https://github.com/penpot/penpot/issues/615)
@@ -473,7 +846,6 @@
 - Improved MacOS shortcuts and helpers
 - Small changes to shape creation
 
-
 ## 1.0.0-alpha
 
 Initial release

CONTRIBUTING.md

@@ -19,9 +19,9 @@ If you found a bug, please report it, as far as possible with:
 - a browser and the browser version used
 - a dev tools console exception stack trace (if it is available)
 
-If you found a bug that you consider better discuse in private (for
+If you found a bug that you consider better discuss in private (for
 example: security bugs), consider first send an email to
-`info@penpot.app`.
+`support@penpot.app`.
 
 **We don't have formal bug bounty program for security reports; this
 is an open source application and your contribution will be recognized
@@ -54,7 +54,7 @@ We will use the `easy fix` mark for tag for indicate issues that are
 easy for beginners.
 
 
-## Commit Message Guidelines ##
+## Commit Guidelines ##
 
 We have very precise rules over how our git commit messages can be formatted.
 
@@ -78,7 +78,6 @@ Where type is:
 - :ambulance: `:ambulance:` a commit that fixes critical bug
 - :books: `:books:` a commit that improves or adds documentation
 - :construction: `:construction:`: a wip commit
-- :construction_worker: `:construction_worker:` a commit with CI related stuff
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
@@ -91,13 +90,14 @@ More info:
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
-The subject should be:
-
-- Use the imperative mood.
-- Capitalize the first letter.
-- Don't put a period at the end of the subject line.
-- Put a blank line between the subject line and the body.
+Each commit should have:
 
+- A concise subject using imperative mood.
+- The subject should have capitalized the first letter and without
+  period at the end.
+- A blank line between the subject line and the body.
+- An entry on the CHANGES.md file if applicable, referencing the
+  github or taiga issue/user-story using the these same rules.
 
 ## Code of conduct ##
 

README.md

@@ -70,9 +70,9 @@ You can ask and answer questions, have open-ended conversations, and follow alon
 
 ✉️ [Mail us](mailto:info@penpot.app)
 
-💬 [Github discussions](https://github.com/penpot/penpot/discussions)
+💬 [GitHub discussions](https://github.com/penpot/penpot/discussions)
 
-🐞 [Github issues](mailto:info@penpot.apphttps://github.com/penpot/penpot/issues)
+🐞 [GitHub issues](https://github.com/penpot/penpot/issues)
 
 ✍️️ [Gitter](https://gitter.im/penpot/community)
 
@@ -81,7 +81,7 @@ You can ask and answer questions, have open-ended conversations, and follow alon
 You can ask and answer questions, have open-ended conversations, and follow along on decisions affecting the project.
 Would you like to know more about Penpot? We recommend you to visit our youtube channel and learn more about the functionalities and possibilities of Penpot with our video tutorials.
 
-🎞️ [Youtube channel](https://www.youtube.com/channel/UCAqS8G72uv9P5HG1IfgnQ9g)
+🎞️ [YouTube channel](https://www.youtube.com/channel/UCAqS8G72uv9P5HG1IfgnQ9g)
 
 ## License ##
 

backend/build.clj

@@ -0,0 +1,36 @@
+(ns build
+  (:refer-clojure :exclude [compile])
+  (:require
+   [clojure.tools.build.api :as b]
+   [clojure.java.io]))
+
+(def class-dir "target/classes")
+(def basis (b/create-basis {:project "deps.edn"}))
+(def jar-file "target/penpot.jar")
+
+(defn clean [_]
+  (b/delete {:path "target"}))
+
+(defn jar [_]
+  (b/copy-dir
+   {:src-dirs ["src" "resources"]
+    :target-dir class-dir})
+
+  (b/compile-clj
+   {:basis basis
+    :src-dirs ["src"]
+    :class-dir class-dir})
+
+  (b/uber
+   {:class-dir class-dir
+    :uber-file jar-file
+    :main 'clojure.main
+    :exclude [#"goog.*" #"^javasist.*"]
+    :basis basis}))
+
+(defn compile [_]
+  (b/javac
+   {:src-dirs ["dev/java"]
+    :class-dir class-dir
+    :basis basis
+    :javac-opts ["-source" "11" "-target" "11"]}))

backend/deps.edn

@@ -1,78 +1,73 @@
-{
- ;; :mvn/repos
- ;; {"central" {:url "https://repo1.maven.org/maven2/"}
- ;;  "clojars" {:url "https://clojars.org/repo"}
- ;;  "jcenter" {:url "https://jcenter.bintray.com/"}
- ;;  }
- :deps
- {penpot/common
-  {:local/root "../common"}
+{:deps
+ {penpot/common {:local/root "../common"}
+  org.clojure/core.async {:mvn/version "1.5.648"}
 
   ;; Logging
   org.zeromq/jeromq {:mvn/version "0.5.2"}
 
   com.taoensso/nippy {:mvn/version "3.1.1"}
-  com.github.luben/zstd-jni {:mvn/version "1.4.9-5"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.2-1"}
+  org.clojure/data.fressian {:mvn/version "1.0.0"}
 
-  ;; NOTE: don't upgrade to latest version, breaking change is
-  ;; introduced on 0.10.0 that suffixes counters with _total if they
-  ;; are not already has this suffix.
-  io.prometheus/simpleclient {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_hotspot {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_jetty {:mvn/version "0.9.0"
+  io.prometheus/simpleclient {:mvn/version "0.14.1"}
+  io.prometheus/simpleclient_hotspot {:mvn/version "0.14.1"}
+  io.prometheus/simpleclient_jetty {:mvn/version "0.14.1"
                                     :exclusions [org.eclipse.jetty/jetty-server
                                                  org.eclipse.jetty/jetty-servlet]}
-  io.prometheus/simpleclient_httpserver {:mvn/version "0.9.0"}
+  io.prometheus/simpleclient_httpserver {:mvn/version "0.14.1"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.1.2.RELEASE"}
-  java-http-clj/java-http-clj {:mvn/version "0.4.2"}
+  io.lettuce/lettuce-core {:mvn/version "6.1.6.RELEASE"}
+  java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
-  info.sunng/ring-jetty9-adapter {:mvn/version "0.15.1"}
-  com.github.seancorfield/next.jdbc {:mvn/version "1.2.659"}
-  metosin/reitit-ring {:mvn/version "0.5.13"}
-  org.postgresql/postgresql {:mvn/version "42.2.20"}
-  com.zaxxer/HikariCP {:mvn/version "4.0.3"}
+  funcool/yetti {:git/tag "v4.0" :git/sha "59ed2a7"
+                 :git/url "https://github.com/funcool/yetti.git"
+                 :exclusions [org.slf4j/slf4j-api]}
 
+  com.github.seancorfield/next.jdbc {:mvn/version "1.2.761"}
+  metosin/reitit-ring {:mvn/version "0.5.15"}
+  org.postgresql/postgresql {:mvn/version "42.3.2"}
+  com.zaxxer/HikariCP {:mvn/version "5.0.1"}
   funcool/datoteka {:mvn/version "2.0.0"}
 
-  buddy/buddy-core {:mvn/version "1.10.1"}
-  buddy/buddy-hashers {:mvn/version "1.8.1"}
-  buddy/buddy-sign {:mvn/version "3.4.1"}
+  buddy/buddy-hashers {:mvn/version "1.8.158"}
+  buddy/buddy-sign {:mvn/version "3.4.333"}
 
-  org.jsoup/jsoup {:mvn/version "1.13.1"}
+  org.jsoup/jsoup {:mvn/version "1.14.3"}
   org.im4java/im4java {:mvn/version "1.4.0"}
-  org.lz4/lz4-java {:mvn/version "1.7.1"}
+  org.lz4/lz4-java {:mvn/version "1.8.0"}
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
   integrant/integrant {:mvn/version "0.8.0"}
 
-  software.amazon.awssdk/s3 {:mvn/version "2.16.62"}}
+  io.sentry/sentry {:mvn/version "5.6.1"}
 
- :paths ["src" "resources"]
+  ;; Pretty Print specs
+  pretty-spec/pretty-spec {:mvn/version "0.1.4"}
+  software.amazon.awssdk/s3 {:mvn/version "2.17.122"}}
+
+ :paths ["src" "resources" "target/classes"]
  :aliases
  {:dev
   {:extra-deps
    {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
     org.clojure/tools.namespace {:mvn/version "RELEASE"}
     org.clojure/test.check {:mvn/version "RELEASE"}
-    com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.0"}
-
-    criterium/criterium {:mvn/version "RELEASE"}
+    clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
+    org.clojure/data.csv {:mvn/version "RELEASE"}
+    com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
     mockery/mockery {:mvn/version "RELEASE"}}
    :extra-paths ["test" "dev"]}
 
-  :fn-fixtures
-  {:exec-fn app.cli.fixtures/run
-   :args {}}
-
-  :kaocha
-  {:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.829"}}
-   :main-opts ["-m" "kaocha.runner"]}
+  :build
+  {:extra-deps
+   {io.github.clojure/tools.build {:git/tag "v0.7.5" :git/sha "34727f7"}}
+   :ns-default build}
 
   :test
-  {:extra-deps {io.github.cognitect-labs/test-runner
-                {:git/url "https://github.com/cognitect-labs/test-runner.git"
-                 :sha "705ad25bbf0228b1c38d0244a36001c2987d7337"}}
+  {:extra-paths ["test"]
+   :extra-deps
+   {io.github.cognitect-labs/test-runner
+    {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
    :exec-fn cognitect.test-runner.api/test}
 
   :outdated

backend/dev/user.clj

@@ -6,13 +6,19 @@
 
 (ns user
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.geom.matrix :as gmt]
+   [app.common.perf :as perf]
+   [app.common.transit :as t]
    [app.config :as cfg]
    [app.main :as main]
    [app.util.blob :as blob]
+   [app.util.fressian :as fres]
    [app.util.json :as json]
    [app.util.time :as dt]
-   [app.util.transit :as t]
+   [clj-async-profiler.core :as prof]
+   [clojure.contrib.humanize :as hum]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]
@@ -22,31 +28,14 @@
    [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
-   [criterium.core :refer [quick-bench bench with-progress-reporting]]
+   [datoteka.core]
    [integrant.core :as ig]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
+(set! *warn-on-reflection* true)
 
 (defonce system nil)
 
-;; --- Benchmarking Tools
-
-(defmacro run-quick-bench
-  [& exprs]
-  `(with-progress-reporting (quick-bench (do ~@exprs) :verbose)))
-
-(defmacro run-quick-bench'
-  [& exprs]
-  `(quick-bench (do ~@exprs)))
-
-(defmacro run-bench
-  [& exprs]
-  `(with-progress-reporting (bench (do ~@exprs) :verbose)))
-
-(defmacro run-bench'
-  [& exprs]
-  `(bench (do ~@exprs)))
-
 ;; --- Development Stuff
 
 (defn- run-tests
@@ -91,7 +80,16 @@
 
 (defn compression-bench
   [data]
-  (print-table
-   [{:v1 (alength (blob/encode data {:version 1}))
-     :v2 (alength (blob/encode data {:version 2}))
-     :v3 (alength (blob/encode data {:version 3}))}]))
+  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))]
+    (print-table
+     [{:v1 (humanize (alength (blob/encode data {:version 1})))
+       :v2 (humanize (alength (blob/encode data {:version 2})))
+       :v3 (humanize (alength (blob/encode data {:version 3})))
+       :v4 (humanize (alength (blob/encode data {:version 4})))
+       }])))
+
+(defonce debug-tap
+  (do
+    (add-tap #(locking debug-tap
+                (prn "tap debug:" %)))
+    1))

backend/resources/api-doc.css

@@ -0,0 +1,101 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+pre {
+  margin: 0px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+  padding-top: 20px;
+  padding-bottom: 20px;
+  display: flex;
+  justify-content: center;
+}
+
+main {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  min-width: 900px;
+  width: 900px;
+}
+
+header {
+  border-bottom: 1px solid #c0c0c0;
+  display: flex;
+  justify-content: center;
+  width: 100%;
+}
+
+.rpc-doc-content {
+  margin-top: 20px;
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  /* border: 1px solid red; */
+  padding: 5px;
+}
+
+.rpc-doc-content > h2:not(:first-child) {
+  margin-top: 30px;
+}
+
+
+.rpc-items {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+}
+
+.rpc-item {
+  /* border: 1px solid red; */
+  cursor: pointer;
+  display: flex;
+  flex-direction: column;
+}
+
+.rpc-item:not(:last-child) {
+  margin-bottom: 3px;
+}
+
+.rpc-row-info {
+  cursor: pointer;
+  display: flex;
+  background-color: #eeeeee;
+  padding: 5px 10px;
+}
+
+.rpc-row-info > *:not(:last-child) {
+  margin-right: 10px;
+}
+
+.rpc-row-info > * {
+  /* border: 1px solid green; */
+}
+
+.rpc-row-info > .type {
+  font-weight: bold;
+  width: 70px;
+}
+
+.rpc-row-info > .name {
+  width: 280px;
+  /* font-weight: bold; */
+}
+
+.rpc-row-info > .tags > .tag > span:first-child {
+  font-weight: bold;
+}
+
+.hidden {
+  display: none;
+}
+
+.rpc-row-detail {
+  padding: 5px 10px;
+  padding-bottom: 20px;
+}

backend/resources/api-doc.js

@@ -0,0 +1,27 @@
+(function() {
+  document.addEventListener("DOMContentLoaded", function(event) {
+    const rows = document.querySelectorAll(".rpc-row-info");
+
+    const onRowClick = (event) => {
+      const target = event.currentTarget;
+      for (let node of rows) {
+        if (node !== target) {
+          node.nextElementSibling.classList.add("hidden");
+        } else {
+          const sibling = target.nextElementSibling;
+
+          if (sibling.classList.contains("hidden")) {
+            sibling.classList.remove("hidden");
+          } else {
+            sibling.classList.add("hidden");
+          }
+        }
+      }
+    };
+
+    for (let node of rows) {
+      node.addEventListener("click", onRowClick);
+    }
+
+  });
+})();

backend/resources/api-doc.tmpl

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Builtin API Documentation - Penpot</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      {% include "api-doc.css" %}
+    </style>
+    <script>
+      {% include "api-doc.js" %}
+    </script>
+  </head>
+  <body>
+    <main>
+      <header>
+        <h1>Penpot API Documentation</h1>
+      </header>
+      <section class="rpc-doc-content">
+
+        <h2>RPC QUERY METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in query-methods %}
+            <li class="rpc-item">
+              <div class="rpc-row-info">
+                {# <div class="type">{{item.type}}</div> #}
+                <div class="name">{{item.name}}</div>
+                <div class="tags">
+                  <span class="tag">
+                    <span>Auth:</span>
+                    <span>{% if item.auth %}YES{% else %}NO{% endif %}</span>
+                  </span>
+                </div>
+              </div>
+              <div class="rpc-row-detail hidden">
+                {% if item.docs %}
+                  <h3>DOCSTRING:</h3>
+                  <p>{{item.docs}}</p>
+                {% endif %}
+
+                <h3>SPEC EXPLAIN:</h3>
+                <pre>{{item.spec}}</pre>
+              </div>
+            </li>
+          {% endfor %}
+        </ul>
+
+        <h2>RPC MUTATION METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in mutation-methods %}
+            <li class="rpc-item">
+              <div class="rpc-row-info">
+                {# <div class="type">{{item.type}}</div> #}
+                <div class="name">{{item.name}}</div>
+                <div class="tags">
+                  <span class="tag">
+                    <span>Auth:</span>
+                    <span>{% if item.auth %}YES{% else %}NO{% endif %}</span>
+                  </span>
+                </div>
+              </div>
+              <div class="rpc-row-detail hidden">
+                {% if item.docs %}
+                  <h3>DOCSTRING:</h3>
+                  <p>{{item.docs}}</p>
+                {% endif %}
+
+                <h3>SPEC EXPLAIN:</h3>
+                <pre>{{item.spec}}</pre>
+              </div>
+            </li>
+          {% endfor %}
+        </ul>
+      </section>
+    </main>
+  </body>
+</html>
+

backend/resources/emails-mjml/register/en.mjml

@@ -22,7 +22,7 @@
         <mj-text font-size="24px" font-weight="600">Hello {{name}}!</mj-text>
         <mj-text>
           Thanks for signing up for your Penpot account! Please verify your
-          email using the link below adn get started building mockups and
+          email using the link below and get started building mockups and
           prototypes today!
         </mj-text>
         <mj-button href="{{ public-uri }}/#/auth/verify-token?token={{token}}">

backend/resources/emails/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{subject|abbreviate:19}} (from {{email}})
+[PENPOT FEEDBACK]: {{subject}}

backend/resources/emails/register/en.html

@@ -173,7 +173,7 @@
                   </tr>
                   <tr>
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below adn get started building mockups and prototypes today!</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below and get started building mockups and prototypes today!</div>
                     </td>
                   </tr>
                   <tr>
@@ -465,4 +465,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/emails/register/en.txt

@@ -1,7 +1,7 @@
 Hello {{name}}!
 
 Thanks for signing up for your Penpot account! Please verify your email using the
-link below adn get started building mockups and prototypes today!
+link below and get started building mockups and prototypes today!
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 

backend/resources/error-report.tmpl

@@ -1,194 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>penpot - error report {{id}}</title>
-    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
-    <style>
-      body {
-        margin: 0px;
-        padding: 0px;
-      }
-      pre {
-        margin: 0px;
-      }
-      * {
-        font-family: "JetBrains Mono", monospace; 
-        font-size: 12px;
-      }
-      .table {
-        display: flex;
-        flex-direction: column;
-        margin: 10px;
-      }
-
-      .table-row {
-        display: flex;
-        /* width: 100%; */
-        /* border: 1px solid red; */
-      }
-
-      .table-key {
-        font-weight: 600;
-        width: 60px;
-        padding: 4px;
-      }
-
-      .table-val {
-        font-weight: 200;
-        color: #333;
-        padding: 4px;
-      }
-
-      .multiline {
-        margin-top: 15px;
-        flex-direction: column;
-      }
-
-      .multiline .table-key {
-        margin-bottom: 10px;
-        border-bottom: 1px dashed #dddddd;
-        /* padding: 4px; */
-        width: unset;
-      }
-
-    </style>
-  </head>
-  <body>
-    <div class="table">
-      <div class="table-row">
-        <div class="table-key" title="Error ID">ERID: </div>
-        <div class="table-val">{{id}}</div>
-      </div>
-      {% if profile-id %}
-      <div class="table-row">
-        <div class="table-key" title="Profile ID">PFID: </div>
-        <div class="table-val">{{profile-id}}</div>
-      </div>
-      {% endif %}
-
-      {% if user-agent %}
-      <div class="table-row">
-        <div class="table-key">UAGT: </div>
-        <div class="table-val">{{user-agent}}</div>
-      </div>
-      {% endif %}
-
-      {% if frontend-version %}
-      <div class="table-row">
-        <div class="table-key">FVER: </div>
-        <div class="table-val">{{frontend-version}}</div>
-      </div>
-      {% endif %}
-
-      <div class="table-row">
-        <div class="table-key">BVER: </div>
-        <div class="table-val">{{version}}</div>
-      </div>
-
-      {% if host %}
-      <div class="table-row">
-        <div class="table-key">HOST: </div>
-        <div class="table-val">{{host}}</div>
-      </div>
-      {% endif %}
-
-      {% if tenant %}
-      <div class="table-row">
-        <div class="table-key">ENV: </div>
-        <div class="table-val">{{tenant}}</div>
-      </div>
-      {% endif %}
-
-      {% if public-uri %}
-      <div class="table-row">
-        <div class="table-key">PURI: </div>
-        <div class="table-val">{{public-uri}}</div>
-      </div>
-      {% endif %}
-
-      {% if type %}
-      <div class="table-row">
-        <div class="table-key">TYPE: </div>
-        <div class="table-val">{{type}}</div>
-      </div>
-      {% endif %}
-
-      {% if code %}
-      <div class="table-row">
-        <div class="table-key">CODE: </div>
-        <div class="table-val">{{code}}</div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div class="table-row">
-        <div class="table-key">CLSS: </div>
-        <div class="table-val">{{error.class}}</div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div class="table-row">
-        <div class="table-key">HINT: </div>
-        <div class="table-val">{{error.message}}</div>
-      </div>
-      {% endif %}
-
-      {% if method %}
-      <div class="table-row">
-        <div class="table-key">PATH: </div>
-        <div class="table-val">{{method|upper}} {{path}}</div>
-      </div>
-      {% endif %}
-
-      {% if explain %}
-        <div>(<a href="#explain">go to explain</a>)</div>
-      {% endif %}
-      {% if data %}
-        <div>(<a href="#edata">go to edata</a>)</div>
-      {% endif %}
-      {% if error %}
-        <div>(<a href="#trace">go to trace</a>)</div>
-      {% endif %}
-
-      {% if params %}
-      <div id="params" class="table-row multiline">
-        <div class="table-key">PARAMS: </div>
-        <div class="table-val">
-          <pre>{{params}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if explain %}
-      <div id="explain" class="table-row multiline">
-        <div class="table-key">EXPLAIN: </div>
-        <div class="table-val">
-          <pre>{{explain}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if data %}
-      <div id="edata" class="table-row multiline">
-        <div class="table-key">EDATA: </div>
-        <div class="table-val">
-          <pre>{{data}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div id="trace" class="table-row multiline">
-        <div class="table-key">TRACE:</div>
-        <div class="table-val">
-          <pre>{{error.trace}}</pre>
-        </div>
-      </div>
-      {% endif %}
-    </div>
-  </body>
-</html>
-

backend/resources/log4j2-devenv.xml

@@ -2,7 +2,7 @@
 <Configuration status="info" monitorInterval="30">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
 
     <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">

backend/resources/log4j2.xml

@@ -2,7 +2,7 @@
 <Configuration status="info" monitorInterval="60">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
   </Appenders>
 

backend/resources/templates/base.tmpl

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>{% block title %}{% endblock %}</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      {% include "templates/styles.css"  %}
+    </style>
+  </head>
+  <body>
+    {% block content %}
+    {% endblock %}
+  </body>
+</html>
+

backend/resources/templates/debug.tmpl

@@ -0,0 +1,32 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+Debug Main Page
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Debug INDEX:</h1>
+  <div>[<a href="/dbg/error">ERRORS</a>]</div>
+</nav>
+<main class="index">
+  <section>
+    <h2>Download file data:</h2>
+    <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
+    <form method="get" action="/dbg/file/data">
+      <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+      <input type="hidden" name="download" value="1" />
+      <input type="submit" value="Download" />
+    </form>
+  </section>
+
+  <section>
+    <h2>Upload File Data:</h2>
+    <desc>Create a new file on your draft projects using the file downloaded from the previous section.</desc>
+    <form method="post" enctype="multipart/form-data" action="/dbg/file/data">
+      <input type="file" name="file" value="" />
+      <input type="submit" value="Upload" />
+    </form>
+  </section>
+</main>
+{% endblock %}

backend/resources/templates/error-list.tmpl

@@ -0,0 +1,18 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error list
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Latest error reports:</h1>
+</nav>
+<main class="horizontal-list">
+  <ul>
+    {% for item in items %}
+    <li><a href="/dbg/error/{{item.id}}">{{item.created-at}}</a></li>
+    {% endfor %}
+  </ul>
+</main>
+{% endblock %}

backend/resources/templates/error-report.tmpl

@@ -0,0 +1,98 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error report {{id}}
+{% endblock %}
+
+{% block content %}
+<nav>
+  <div>[<a href="/dbg/error">⮜</a>]</div>
+  <div>[<a href="#context">context</a>]</div>
+  <div>[<a href="#params">request params</a>]</div>
+  {% if data %}
+  <div>[<a href="#edata">error data</a>]</div>
+  {% endif %}
+  {% if spec-explain %}
+  <div>[<a href="#spec-explain">spec explain</a>]</div>
+  {% endif %}
+  {% if spec-problems %}
+  <div>[<a href="#spec-problems">spec problems</a>]</div>
+  {% endif %}
+  {% if spec-value %}
+  <div>[<a href="#spec-value">spec value</a>]</div>
+  {% endif %}
+
+  {% if trace %}
+  <div>[<a href="#trace">error trace</a>]</div>
+  {% endif %}
+</nav>
+<main>
+  <div class="table">
+    <div class="table-row multiline">
+      <div id="context" class="table-key">CONTEXT: </div>
+
+      <div class="table-val">
+        <h1>{{hint}}</h1>
+      </div>
+
+      <div class="table-val">
+        <pre>{{context}}</pre>
+      </div>
+    </div>
+
+    {% if params %}
+    <div class="table-row multiline">
+      <div id="params" class="table-key">REQUEST PARAMS: </div>
+      <div class="table-val">
+        <pre>{{params}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if data %}
+    <div class="table-row multiline">
+      <div id="edata" class="table-key">ERROR DATA: </div>
+      <div class="table-val">
+        <pre>{{data}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-explain %}
+    <div class="table-row multiline">
+      <div id="spec-explain" class="table-key">SPEC EXPLAIN: </div>
+      <div class="table-val">
+        <pre>{{spec-explain}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-problems %}
+    <div class="table-row multiline">
+      <div id="spec-problems" class="table-key">SPEC PROBLEMS: </div>
+      <div class="table-val">
+        <pre>{{spec-problems}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-value %}
+    <div class="table-row multiline">
+      <div id="spec-value" class="table-key">SPEC VALUE: </div>
+      <div class="table-val">
+        <pre>{{spec-value}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if trace %}
+    <div class="table-row multiline">
+      <div id="trace" class="table-key">TRACE:</div>
+      <div class="table-val">
+        <pre>{{trace}}</pre>
+      </div>
+    </div>
+    {% endif %}
+  </div>
+</main>
+{% endblock %}

backend/resources/templates/styles.css

@@ -0,0 +1,150 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+}
+
+pre {
+  margin: 0px;
+  line-height: 16px;
+}
+
+desc {
+  display: flex;
+  margin-bottom: 10px;
+  font-size: 10px;
+  color: #666;
+}
+
+input[type=text], input[type=submit] {
+  padding: 3px;
+}
+
+main {
+  margin: 20px;
+}
+
+nav {
+  position: fixed;
+  width: 100vw;
+  top: 0;
+  left: 0;
+  padding: 5px 20px;
+  display: flex;
+  background: #e3e3e3;
+}
+
+nav > h1 {
+  padding: 0px;
+  margin: 0px;
+  font-size: 11px;
+}
+
+nav > div {
+  text-transform: uppercase;
+  font-weight: bold;
+}
+
+nav > div:not(:last-child) {
+  margin-right: 10px;
+}
+
+.table {
+  margin-top: 25px;
+  display: flex;
+  flex-direction: column;
+}
+
+.table-row {
+  display: flex;
+  padding-bottom: 15px;
+  /* width: 100%; */
+  /* border: 1px solid red; */
+}
+
+.table-key {
+  font-weight: 600;
+  width: 60px;
+  padding: 4px;
+
+  padding-top: 40px;
+  margin-top: -40px;
+}
+
+.table-val {
+  font-weight: 200;
+  color: #333;
+  padding: 4px;
+}
+
+.multiline {
+  margin-top: 15px;
+  flex-direction: column;
+}
+
+.multiline .table-key {
+  margin-bottom: 10px;
+  border-bottom: 1px dashed #dddddd;
+  /* padding: 4px; */
+  width: unset;
+}
+
+.index {
+  margin-top: 40px;
+}
+
+.index > section {
+  padding: 10px;
+  background-color: #e3e3e3;
+}
+
+.index > section:not(:last-child) {
+  margin-bottom: 10px;
+}
+
+
+.index > section > h2 {
+  margin-top: 0px;
+}
+
+.horizontal-list {
+  margin: 20px;
+  margin-top: 40px;
+}
+
+.horizontal-list ul {
+  display: flex;
+  margin: 0px;
+  padding: 0px;
+  flex-direction: column;
+  flex-wrap: wrap;
+  height: calc(100vh - 75px);
+  justify-content: flex-start;
+}
+
+.horizontal-list li {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+  line-height: 18px;
+  min-width: 210px;
+  margin: 0px 20px;
+  cursor: pointer;
+  display: flex;
+  justify-content: center;
+  border-radius: 3px;
+}
+
+.horizontal-list li:hover {
+  background-color: #e9e9e9;
+}
+
+.horizontal-list li > a {
+  text-decoration: none;
+  color: inherit;
+}
+

backend/scripts/build

@@ -1,79 +1,20 @@
-#!/usr/bin/env bb
+#!/usr/bin/env bash
 
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
+CURRENT_VERSION=$1;
 
-(ns build
-  (:require
-   [clojure.string :as str]
-   [clojure.java.io :as io]
-   [clojure.pprint :refer [pprint]]
-   [babashka.fs :as fs]
-   [babashka.process :refer [$ check]]))
+set -ex
 
-(defn split-cp
-  [data]
-  (str/split data #":"))
+rm -rf target;
+mkdir -p target/classes;
+mkdir -p target/dist;
+echo "$CURRENT_VERSION" > target/classes/version.txt;
 
-(def classpath
-  (->> ($ clojure -Spath)
-       (check)
-       (:out)
-       (slurp)
-       (split-cp)
-       (map str/trim)))
+clojure -T:build jar;
+mv target/penpot.jar target/dist/penpot.jar
+cp scripts/run.template.sh target/dist/run.sh;
+cp scripts/manage.template.sh target/dist/manage.sh;
+chmod +x target/dist/run.sh;
+chmod +x target/dist/manage.sh;
 
-(def classpath-jars
-  (let [xfm (filter #(str/ends-with? % ".jar"))]
-    (into #{} xfm classpath)))
 
-(def classpath-paths
-  (let [xfm (comp (remove #(str/ends-with? % ".jar"))
-                  (filter #(.isDirectory (io/file %))))]
-    (into #{} xfm classpath)))
 
-(def version
-  (or (first *command-line-args*) "%version%"))
-
-;; Clean previous dist
-(-> ($ rm -rf "./target/dist") check)
-
-;; Create a new dist
-(-> ($ mkdir -p "./target/dist/deps") check)
-
-;; Copy all jar deps into dist
-(run! (fn [item] (-> ($ cp ~item "./target/dist/deps/") check)) classpath-jars)
-
-;; Create the application jar
-(spit "./target/dist/version.txt" version)
-
-(-> ($ jar cvf "./target/dist/deps/app.jar" -C ~(first classpath-paths) ".") check)
-(-> ($ jar uvf "./target/dist/deps/app.jar" -C "./target/dist" "version.txt") check)
-(run! (fn [item]
-        (-> ($ jar uvf "./target/dist/deps/app.jar" -C ~item ".")  check))
-      (rest classpath-paths))
-
-;; Copy logging configuration
-(-> ($ cp "./resources/log4j2.xml" "./target/dist/") check)
-
-;; Create classpath file
-(let [jars (->> (into ["app.jar"] classpath-jars)
-                (map fs/file-name)
-                (map #(fs/path "deps" %))
-                (map str))]
-  (spit "./target/dist/classpath" (str/join ":" jars)))
-
-;; Copy run script template
-(-> ($ cp "./scripts/run.template.sh" "./target/dist/run.sh") check)
-
-;; Copy run script template
-(-> ($ cp "./scripts/manage.template.sh" "./target/dist/manage.sh") check)
-
-;; Add exec permisions to scripts.
-(-> ($ chmod +x "./target/dist/run.sh") check)
-(-> ($ chmod +x "./target/dist/manage.sh") check)
-
-nil

backend/scripts/manage.template.sh

@@ -16,4 +16,4 @@ if [ -f ./environ ]; then
    source ./environ
 fi
 
-exec $JAVA_CMD $JVM_OPTS -classpath $(cat classpath) -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "$@"
+exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m app.cli.manage "$@"

backend/scripts/repl

@@ -1,10 +1,37 @@
 #!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
+# export PENPOT_DATABASE_USERNAME="penpot"
+# export PENPOT_DATABASE_PASSWORD="penpot"
+# export PENPOT_DATABASE_READONLY=true
 
-export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-Djdk.attach.allowAttachSelf -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
-# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";
-# export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
+# export PENPOT_DATABASE_USERNAME="penpot_pre"
+# export PENPOT_DATABASE_PASSWORD="penpot_pre"
+# export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS"
+
+# Initialize MINIO config
+# mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
+# mc admin user add penpot-s3 penpot-devenv penpot-devenv
+# mc admin policy set penpot-s3 readwrite user=penpot-devenv
+# mc mb penpot-s3/penpot -p
+# export AWS_ACCESS_KEY_ID=penpot-devenv
+# export AWS_SECRET_ACCESS_KEY=penpot-devenv
+# export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
+# export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
+# export PENPOT_STORAGE_ASSETS_S3_REGION=eu-central-1
+# export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
+
+export OPTIONS="
+       -A:dev \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -J-XX:+UseZGC \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-Xms50m -J-Xmx1024m \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints";
 
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"

backend/scripts/run.template.sh

@@ -17,4 +17,4 @@ if [ -f ./environ ]; then
 fi
 
 set -x
-exec $JAVA_CMD $JVM_OPTS -classpath "$(cat classpath)" -Dlog4j2.configurationFile=./log4j2.xml "$@" clojure.main -m app.main
+exec $JAVA_CMD $JVM_OPTS "$@" -jar penpot.jar -m app.main

backend/scripts/start-dev

@@ -1,15 +1,23 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+export PENPOT_FLAGS="$PENPOT_FLAGS enable-asserts"
 
 set -ex
 
-if [ ! -e ~/.fixtures-loaded ]; then
-    echo "Loading fixtures..."
-    clojure -Adev -X:fn-fixtures
-    touch ~/.fixtures-loaded
+if [ "$1" = "--watch" ]; then
+    echo "Start Watch..."
+
+    clojure -A:dev -M -m app.main &
+    PID=$!
+
+    npx nodemon \
+        --watch src \
+        --watch ../common \
+        --ext "clj" \
+        --signal SIGKILL \
+        --exec 'echo "(user/restart)" | nc -N localhost 6062'
+
+    kill -9 $PID
+else
+    clojure -A:dev -M -m app.main
 fi
-
-clojure -A:dev -M -m app.main
-
-

backend/src/app/cli/fixtures.clj

@@ -1,258 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.cli.fixtures
-  "A initial fixtures."
-  (:require
-   [app.common.pages :as cp]
-   [app.common.uuid :as uuid]
-   [app.db :as db]
-   [app.main :as main]
-   [app.rpc.mutations.profile :as profile]
-   [app.util.blob :as blob]
-   [app.util.logging :as l]
-   [buddy.hashers :as hashers]
-   [integrant.core :as ig]))
-
-(defn- mk-uuid
-  [prefix & args]
-  (uuid/namespaced uuid/zero (apply str prefix (interpose "-" args))))
-
-;; --- Profiles creation
-
-(def password (hashers/derive "123123"))
-
-(def preset-small
-  {:num-teams 5
-   :num-profiles 5
-   :num-profiles-per-team 5
-   :num-projects-per-team 5
-   :num-files-per-project 5
-   :num-draft-files-per-profile 10})
-
-(defn- rng-ids
-  [rng n max]
-  (let [stream (->> (.longs rng 0 max)
-                    (.iterator)
-                    (iterator-seq))]
-    (reduce (fn [acc item]
-              (if (= (count acc) n)
-                (reduced acc)
-                (conj acc item)))
-            #{}
-            stream)))
-
-(defn- rng-vec
-  [rng vdata n]
-  (let [ids (rng-ids rng n (count vdata))]
-    (mapv #(nth vdata %) ids)))
-
-(defn- rng-nth
-  [rng vdata]
-  (let [stream (->> (.longs rng 0 (count vdata))
-                    (.iterator)
-                    (iterator-seq))]
-    (nth vdata (first stream))))
-
-(defn- collect
-  [f items]
-  (reduce #(conj %1 (f %2)) [] items))
-
-(defn- register-profile
-  [conn params]
-  (->> (#'profile/create-profile conn params)
-       (#'profile/create-profile-relations conn)))
-
-(defn impl-run
-  [pool opts]
-  (let [rng (java.util.Random. 1)]
-    (letfn [(create-profile [conn index]
-              (let [id   (mk-uuid "profile" index)
-                    _    (l/info :action "create profile"
-                                 :index index
-                                 :id id)
-
-                    prof (register-profile conn
-                                           {:id id
-                                            :fullname (str "Profile " index)
-                                            :password "123123"
-                                            :is-demo true
-                                            :email (str "profile" index "@example.com")})
-                    team-id  (:default-team-id prof)
-                    owner-id id]
-                (let [project-ids (collect (partial create-project conn team-id owner-id)
-                                           (range (:num-projects-per-team opts)))]
-                  (run! (partial create-files conn owner-id) project-ids))
-                prof))
-
-            (create-profiles [conn]
-              (l/info :action "create profiles")
-              (collect (partial create-profile conn)
-                       (range (:num-profiles opts))))
-
-            (create-team [conn index]
-              (let [id (mk-uuid "team" index)
-                    name (str "Team" index)]
-                (l/info :action "create team"
-                        :index index
-                        :id id)
-                (db/insert! conn :team {:id id
-                                        :name name})
-                id))
-
-            (create-teams [conn]
-              (l/info :action "create teams")
-              (collect (partial create-team conn)
-                       (range (:num-teams opts))))
-
-            (create-file [conn owner-id project-id index]
-              (let [id (mk-uuid "file" project-id index)
-                    name (str "file" index)
-                    data (cp/make-file-data id)]
-                (l/info :action "create file"
-                        :index index
-                        :id id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-files [conn owner-id project-id]
-              (l/info :action "create files")
-              (run! (partial create-file conn owner-id project-id)
-                    (range (:num-files-per-project opts))))
-
-            (create-project [conn team-id owner-id index]
-              (let [id        (if index
-                                (mk-uuid "project" team-id index)
-                                (mk-uuid "project" team-id))
-                    name      (if index
-                                (str "project " index)
-                                "Drafts")
-                    is-default (nil? index)]
-                (l/info :action "create project"
-                        :index index
-                        :id id)
-                (db/insert! conn :project
-                            {:id id
-                             :team-id team-id
-                             :is-default is-default
-                             :name name})
-                (db/insert! conn :project-profile-rel
-                            {:project-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-projects [conn team-id profile-ids]
-              (l/info :action "create projects")
-              (let [owner-id (rng-nth rng profile-ids)
-                    project-ids (conj
-                                  (collect (partial create-project conn team-id owner-id)
-                                         (range (:num-projects-per-team opts)))
-                                  (create-project conn team-id owner-id nil))]
-                (run! (partial create-files conn owner-id) project-ids)))
-
-            (assign-profile-to-team [conn team-id owner? profile-id]
-              (db/insert! conn :team-profile-rel
-                          {:team-id team-id
-                           :profile-id profile-id
-                           :is-owner owner?
-                           :is-admin true
-                           :can-edit true}))
-
-            (setup-team [conn team-id profile-ids]
-              (l/info :action "setup team"
-                      :team-id team-id
-                      :profile-ids (pr-str profile-ids))
-              (assign-profile-to-team conn team-id true (first profile-ids))
-              (run! (partial assign-profile-to-team conn team-id false)
-                    (rest profile-ids))
-              (create-projects conn team-id profile-ids))
-
-            (assign-teams-and-profiles [conn teams profiles]
-              (l/info :action "assign teams and profiles")
-              (loop [team-id (first teams)
-                     teams (rest teams)]
-                (when-not (nil? team-id)
-                  (let [n-profiles-team (:num-profiles-per-team opts)
-                        selected-profiles (rng-vec rng profiles n-profiles-team)]
-                    (setup-team conn team-id selected-profiles)
-                    (recur (first teams)
-                           (rest teams))))))
-
-            (create-draft-file [conn owner index]
-              (let [owner-id   (:id owner)
-                    id         (mk-uuid "file" "draft" owner-id index)
-                    name       (str "file" index)
-                    project-id (:default-project-id owner)
-                    data       (cp/make-file-data id)]
-
-                (l/info :action "create draft file"
-                        :index index
-                        :id id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-draft-files [conn profile]
-              (run! (partial create-draft-file conn profile)
-                    (range (:num-draft-files-per-profile opts))))
-            ]
-      (db/with-atomic [conn pool]
-        (let [profiles (create-profiles conn)
-              teams    (create-teams conn)]
-          (assign-teams-and-profiles conn teams (map :id profiles))
-          (run! (partial create-draft-files conn) profiles))))))
-
-(defn run-in-system
-  [system preset]
-  (let [pool   (:app.db/pool system)
-        preset (if (map? preset)
-                 preset
-                 (case preset
-                   (nil "small" :small) preset-small
-                   ;; "medium" preset-medium
-                   ;; "big" preset-big
-                   preset-small))]
-    (impl-run pool preset)))
-
-(defn run
-  [{:keys [preset] :or {preset :small}}]
-  (let [config (select-keys main/system-config
-                            [:app.db/pool
-                             :app.telemetry/migrations
-                             :app.migrations/migrations
-                             :app.migrations/all
-                             :app.metrics/metrics])
-        _      (ig/load-namespaces config)
-        system (-> (ig/prep config)
-                   (ig/init))]
-    (try
-      (run-in-system system preset)
-      (catch Exception e
-        (l/error :hint "unhandled exception" :cause e))
-      (finally
-        (ig/halt! system)))))

backend/src/app/cli/manage.clj

@@ -7,11 +7,11 @@
 (ns app.cli.manage
   "A manage cli api."
   (:require
+   [app.common.logging :as l]
    [app.db :as db]
    [app.main :as main]
    [app.rpc.mutations.profile :as profile]
    [app.rpc.queries.profile :refer [retrieve-profile-data-by-email]]
-   [app.util.logging :as l]
    [clojure.string :as str]
    [clojure.tools.cli :refer [parse-opts]]
    [integrant.core :as ig])
@@ -140,7 +140,6 @@
   indicating the action the program should take and the options provided."
   [args]
   (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)]
-    ;; (pp/pprint opts)
     (cond
       (:help options) ; help => exit OK with usage summary
       {:exit-message (usage summary) :ok? true}

backend/src/app/cli/migrate_media.clj

@@ -6,12 +6,12 @@
 
 (ns app.cli.migrate-media
   (:require
+   [app.common.logging :as l]
    [app.common.media :as cm]
    [app.config :as cf]
    [app.db :as db]
    [app.main :as main]
    [app.storage :as sto]
-   [app.util.logging :as l]
    [cuerdas.core :as str]
    [datoteka.core :as fs]
    [integrant.core :as ig]))

backend/src/app/config.clj

@@ -10,6 +10,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.flags :as flags]
    [app.common.spec :as us]
    [app.common.version :as v]
    [app.util.time :as dt]
@@ -40,8 +41,7 @@
     data))
 
 (def defaults
-  {:http-server-port 6060
-   :host "devenv"
+  {:host "devenv"
    :tenant "dev"
    :database-uri "postgresql://postgres/penpot"
    :database-username "penpot"
@@ -50,7 +50,8 @@
    :default-blob-version 3
    :loggers-zmq-uri "tcp://localhost:45556"
 
-   :asserts-enabled false
+   :file-change-snapshot-every 5
+   :file-change-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
    :redis-uri "redis://redis/0"
@@ -61,15 +62,12 @@
    :assets-storage-backend :assets-fs
    :storage-assets-fs-directory "assets"
 
-   :feedback-destination "info@example.com"
-   :feedback-enabled false
-
    :assets-path "/internal/assets/"
 
-   :rlimits-password 10
-   :rlimits-image 2
+   :rlimit-password 10
+   :rlimit-image 2
+   :rlimit-font 5
 
-   :smtp-enabled false
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
@@ -79,10 +77,6 @@
    :profile-bounce-max-age (dt/duration {:days 7})
    :profile-bounce-threshold 10
 
-   :allow-demo-users true
-   :registration-enabled true
-
-   :telemetry-enabled false
    :telemetry-uri "https://telemetry.penpot.app/"
 
    :ldap-user-query "(|(uid=:username)(mail=:username))"
@@ -92,27 +86,42 @@
    :ldap-attrs-photo "jpegPhoto"
 
    ;; a server prop key where initial project is stored.
-   :initial-project-skey "initial-project"
-   })
+   :initial-project-skey "initial-project"})
 
-(s/def ::audit-enabled ::us/boolean)
-(s/def ::audit-archive-enabled ::us/boolean)
-(s/def ::audit-archive-uri ::us/string)
-(s/def ::audit-archive-gc-enabled ::us/boolean)
-(s/def ::audit-archive-gc-max-age ::dt/duration)
+(s/def ::flags ::us/set-of-keywords)
+
+;; DEPRECATED PROPERTIES: should be removed in 1.10
+(s/def ::registration-enabled ::us/boolean)
+(s/def ::smtp-enabled ::us/boolean)
+(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::asserts-enabled ::us/boolean)
+;; END DEPRECATED
+
+(s/def ::audit-log-archive-uri ::us/string)
+(s/def ::audit-log-gc-max-age ::dt/duration)
+
+(s/def ::admins ::us/set-of-str)
+(s/def ::file-change-snapshot-every ::us/integer)
+(s/def ::file-change-snapshot-timeout ::dt/duration)
+
+(s/def ::default-executor-parallelism ::us/integer)
+(s/def ::blocking-executor-parallelism ::us/integer)
+(s/def ::worker-executor-parallelism ::us/integer)
 
 (s/def ::secret-key ::us/string)
 (s/def ::allow-demo-users ::us/boolean)
-(s/def ::asserts-enabled ::us/boolean)
 (s/def ::assets-path ::us/string)
+(s/def ::authenticated-cookie-domain ::us/string)
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
 (s/def ::database-username (s/nilable ::us/string))
+(s/def ::database-readonly ::us/boolean)
+(s/def ::database-min-pool-size ::us/integer)
+(s/def ::database-max-pool-size ::us/integer)
+
 (s/def ::default-blob-version ::us/integer)
 (s/def ::error-report-webhook ::us/string)
-(s/def ::feedback-destination ::us/string)
-(s/def ::feedback-enabled ::us/boolean)
-(s/def ::feedback-token ::us/string)
+(s/def ::user-feedback-destination ::us/string)
 (s/def ::github-client-id ::us/string)
 (s/def ::github-client-secret ::us/string)
 (s/def ::gitlab-base-uri ::us/string)
@@ -131,6 +140,9 @@
 (s/def ::oidc-roles-attr ::us/keyword)
 (s/def ::host ::us/string)
 (s/def ::http-server-port ::us/integer)
+(s/def ::http-server-host ::us/string)
+(s/def ::http-server-min-threads ::us/integer)
+(s/def ::http-server-max-threads ::us/integer)
 (s/def ::http-session-idle-max-age ::dt/duration)
 (s/def ::http-session-updater-batch-max-age ::dt/duration)
 (s/def ::http-session-updater-batch-max-size ::us/integer)
@@ -158,12 +170,11 @@
 (s/def ::public-uri ::us/string)
 (s/def ::redis-uri ::us/string)
 (s/def ::registration-domain-whitelist ::us/set-of-str)
-(s/def ::registration-enabled ::us/boolean)
-(s/def ::rlimits-image ::us/integer)
-(s/def ::rlimits-password ::us/integer)
+(s/def ::rlimit-font ::us/integer)
+(s/def ::rlimit-image ::us/integer)
+(s/def ::rlimit-password ::us/integer)
 (s/def ::smtp-default-from ::us/string)
 (s/def ::smtp-default-reply-to ::us/string)
-(s/def ::smtp-enabled ::us/boolean)
 (s/def ::smtp-host ::us/string)
 (s/def ::smtp-password (s/nilable ::us/string))
 (s/def ::smtp-port ::us/integer)
@@ -177,31 +188,42 @@
 (s/def ::storage-assets-fs-directory ::us/string)
 (s/def ::storage-assets-s3-bucket ::us/string)
 (s/def ::storage-assets-s3-region ::us/keyword)
+(s/def ::storage-assets-s3-endpoint ::us/string)
 (s/def ::storage-fdata-s3-bucket ::us/string)
 (s/def ::storage-fdata-s3-region ::us/keyword)
 (s/def ::storage-fdata-s3-prefix ::us/string)
-(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::storage-fdata-s3-endpoint ::us/string)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
 
+(s/def ::sentry-trace-sample-rate ::us/number)
+(s/def ::sentry-attach-stack-trace ::us/boolean)
+(s/def ::sentry-debug ::us/boolean)
+(s/def ::sentry-dsn ::us/string)
+
 (s/def ::config
   (s/keys :opt-un [::secret-key
+                   ::flags
+                   ::admins
                    ::allow-demo-users
-                   ::audit-enabled
-                   ::audit-archive-enabled
-                   ::audit-archive-uri
-                   ::audit-archive-gc-enabled
-                   ::audit-archive-gc-max-age
-                   ::asserts-enabled
+                   ::audit-log-archive-uri
+                   ::audit-log-gc-max-age
+                   ::authenticated-cookie-domain
                    ::database-password
                    ::database-uri
                    ::database-username
+                   ::database-readonly
+                   ::database-min-pool-size
+                   ::database-max-pool-size
                    ::default-blob-version
                    ::error-report-webhook
-                   ::feedback-destination
-                   ::feedback-enabled
-                   ::feedback-token
+                   ::default-executor-parallelism
+                   ::blocking-executor-parallelism
+                   ::worker-executor-parallelism
+                   ::file-change-snapshot-every
+                   ::file-change-snapshot-timeout
+                   ::user-feedback-destination
                    ::github-client-id
                    ::github-client-secret
                    ::gitlab-base-uri
@@ -219,7 +241,10 @@
                    ::oidc-roles-attr
                    ::oidc-roles
                    ::host
+                   ::http-server-host
                    ::http-server-port
+                   ::http-server-max-threads
+                   ::http-server-min-threads
                    ::http-session-idle-max-age
                    ::http-session-updater-batch-max-age
                    ::http-session-updater-batch-max-size
@@ -247,8 +272,13 @@
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::registration-enabled
-                   ::rlimits-image
-                   ::rlimits-password
+                   ::rlimit-font
+                   ::rlimit-image
+                   ::rlimit-password
+                   ::sentry-dsn
+                   ::sentry-debug
+                   ::sentry-attach-stack-trace
+                   ::sentry-trace-sample-rate
                    ::smtp-default-from
                    ::smtp-default-reply-to
                    ::smtp-enabled
@@ -258,26 +288,35 @@
                    ::smtp-ssl
                    ::smtp-tls
                    ::smtp-username
-
                    ::srepl-host
                    ::srepl-port
-
                    ::assets-storage-backend
                    ::storage-assets-fs-directory
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
-
+                   ::storage-assets-s3-endpoint
                    ::fdata-storage-backend
                    ::storage-fdata-s3-bucket
                    ::storage-fdata-s3-region
                    ::storage-fdata-s3-prefix
-
+                   ::storage-fdata-s3-endpoint
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer
                    ::telemetry-with-taiga
                    ::tenant]))
 
+(def default-flags
+  [:enable-backend-asserts
+   :enable-backend-api-doc
+   :enable-secure-session-cookies])
+
+(defn- parse-flags
+  [config]
+  (flags/parse flags/default
+               default-flags
+               (:flags config)))
+
 (defn read-env
   [prefix]
   (let [prefix (str prefix "-")
@@ -304,11 +343,14 @@
         (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
       (throw e))))
 
-(def version (v/parse (or (some-> (io/resource "version.txt")
-                                  (slurp)
-                                  (str/trim))
-                          "%version%")))
-(def config (atom (read-config)))
+(def version
+  (v/parse (or (some-> (io/resource "version.txt")
+                       (slurp)
+                       (str/trim))
+               "%version%")))
+
+(def ^:dynamic config (read-config))
+(def ^:dynamic flags  (parse-flags config))
 
 (def deletion-delay
   (dt/duration {:days 7}))
@@ -316,9 +358,9 @@
 (defn get
   "A configuration getter. Helps code be more testable."
   ([key]
-   (c/get @config key))
+   (c/get config key))
   ([key default]
-   (c/get @config key default)))
+   (c/get config key default)))
 
 ;; Set value for all new threads bindings.
-(alter-var-root #'*assert* (constantly (get :asserts-enabled)))
+(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -9,13 +9,13 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.db.sql :as sql]
    [app.metrics :as mtx]
    [app.util.json :as json]
-   [app.util.logging :as l]
    [app.util.migrations :as mg]
    [app.util.time :as dt]
    [clojure.java.io :as io]
@@ -27,14 +27,16 @@
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
+   java.io.InputStream
+   java.io.OutputStream
    java.lang.AutoCloseable
    java.sql.Connection
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
+   org.postgresql.jdbc.PgArray
    org.postgresql.largeobject.LargeObject
    org.postgresql.largeobject.LargeObjectManager
-   org.postgresql.jdbc.PgArray
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
@@ -45,76 +47,104 @@
 ;; Initialization
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare instrument-jdbc!)
+(declare apply-migrations!)
 
-(s/def ::name keyword?)
-(s/def ::uri ::us/not-empty-string)
-(s/def ::min-pool-size ::us/integer)
-(s/def ::max-pool-size ::us/integer)
+(s/def ::connection-timeout ::us/integer)
+(s/def ::max-size ::us/integer)
+(s/def ::min-size ::us/integer)
 (s/def ::migrations map?)
+(s/def ::name keyword?)
+(s/def ::password ::us/string)
+(s/def ::read-only ::us/boolean)
+(s/def ::uri ::us/not-empty-string)
+(s/def ::username ::us/string)
+(s/def ::validation-timeout ::us/integer)
 
 (defmethod ig/pre-init-spec ::pool [_]
-  (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size ::migrations ::mtx/metrics]))
+  (s/keys :req-un [::uri ::name
+                   ::min-size
+                   ::max-size
+                   ::connection-timeout
+                   ::validation-timeout]
+          :opt-un [::migrations
+                   ::username
+                   ::password
+                   ::mtx/metrics
+                   ::read-only]))
+
+(defmethod ig/prep-key ::pool
+  [_ cfg]
+  (merge {:name :main
+          :min-size 0
+          :max-size 30
+          :connection-timeout 10000
+          :validation-timeout 10000
+          :idle-timeout 120000 ; 2min
+          :max-lifetime 1800000 ; 30m
+          :read-only false}
+         (d/without-nils cfg)))
 
 (defmethod ig/init-key ::pool
-  [_ {:keys [migrations metrics] :as cfg}]
-  (l/info :action "initialize connection pool"
-          :name (d/name (:name cfg))
-          :uri (:uri cfg))
-  (instrument-jdbc! (:registry metrics))
+  [_ {:keys [migrations name read-only] :as cfg}]
+  (l/info :hint "initialize connection pool"
+          :name (d/name name)
+          :uri (:uri cfg)
+          :read-only read-only
+          :with-credentials (and (contains? cfg :username)
+                                 (contains? cfg :password))
+          :min-size (:min-size cfg)
+          :max-size (:max-size cfg))
+
   (let [pool (create-pool cfg)]
-    (when (seq migrations)
-      (with-open [conn ^AutoCloseable (open pool)]
-        (mg/setup! conn)
-        (doseq [[name steps] migrations]
-          (mg/migrate! conn {:name (d/name name) :steps steps}))))
+    (when-not read-only
+      (some->> (seq migrations) (apply-migrations! pool)))
     pool))
 
 (defmethod ig/halt-key! ::pool
   [_ pool]
   (.close ^HikariDataSource pool))
 
-(defn- instrument-jdbc!
-  [registry]
-  (mtx/instrument-vars!
-   [#'next.jdbc/execute-one!
-    #'next.jdbc/execute!]
-   {:registry registry
-    :type :counter
-    :name "database_query_total"
-    :help "An absolute counter of database queries."}))
+(defn- apply-migrations!
+  [pool migrations]
+  (with-open [conn ^AutoCloseable (open pool)]
+    (mg/setup! conn)
+    (doseq [[name steps] migrations]
+      (mg/migrate! conn {:name (d/name name) :steps steps}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; API & Impl
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def initsql
-  (str "SET statement_timeout = 120000;\n"
-       "SET idle_in_transaction_session_timeout = 120000;"))
+  (str "SET statement_timeout = 300000;\n"
+       "SET idle_in_transaction_session_timeout = 300000;"))
 
 (defn- create-datasource-config
-  [{:keys [metrics] :as cfg}]
-  (let [dburi    (:uri cfg)
-        username (:username cfg)
-        password (:password cfg)
-        config   (HikariConfig.)
-        mtf      (PrometheusMetricsTrackerFactory. (:registry metrics))]
+  [{:keys [metrics uri] :as cfg}]
+  (let [config (HikariConfig.)]
     (doto config
-      (.setJdbcUrl (str "jdbc:" dburi))
-      (.setPoolName (d/name (:name cfg)))
+      (.setJdbcUrl           (str "jdbc:" uri))
+      (.setPoolName          (d/name (:name cfg)))
       (.setAutoCommit true)
-      (.setReadOnly false)
-      (.setConnectionTimeout 8000)  ;; 8seg
-      (.setValidationTimeout 8000)  ;; 8seg
-      (.setIdleTimeout 120000)      ;; 2min
-      (.setMaxLifetime 1800000)     ;; 30min
-      (.setMinimumIdle (:min-pool-size cfg 0))
-      (.setMaximumPoolSize (:max-pool-size cfg 30))
-      (.setMetricsTrackerFactory mtf)
+      (.setReadOnly          (:read-only cfg))
+      (.setConnectionTimeout (:connection-timeout cfg))
+      (.setValidationTimeout (:validation-timeout cfg))
+      (.setIdleTimeout       (:idle-timeout cfg))
+      (.setMaxLifetime       (:max-lifetime cfg))
+      (.setMinimumIdle       (:min-size cfg))
+      (.setMaximumPoolSize   (:max-size cfg))
       (.setConnectionInitSql initsql)
       (.setInitializationFailTimeout -1))
-    (when username (.setUsername config username))
-    (when password (.setPassword config password))
+
+    ;; When metrics namespace is provided
+    (when metrics
+      (->> (:registry metrics)
+           (PrometheusMetricsTrackerFactory.)
+           (.setMetricsTrackerFactory config)))
+
+    (some->> ^String (:username cfg) (.setUsername config))
+    (some->> ^String (:password cfg) (.setPassword config))
+
     config))
 
 (defn pool?
@@ -123,11 +153,15 @@
 
 (s/def ::pool pool?)
 
-(defn pool-closed?
+(defn closed?
   [pool]
   (.isClosed ^HikariDataSource pool))
 
-(defn- create-pool
+(defn read-only?
+  [pool]
+  (.isReadOnly ^HikariDataSource pool))
+
+(defn create-pool
   [cfg]
   (let [dsc (create-datasource-config cfg)]
     (jdbc-dt/read-as-instant)
@@ -231,9 +265,9 @@
 (defn get-by-params
   ([ds table params]
    (get-by-params ds table params nil))
-  ([ds table params {:keys [uncheked] :or {uncheked false} :as opts}]
+  ([ds table params {:keys [check-not-found] :or {check-not-found true} :as opts}]
    (let [res (exec-one! ds (sql/select table params opts))]
-     (when (and (not uncheked) (or (not res) (is-deleted? res)))
+     (when (and check-not-found (or (not res) (is-deleted? res)))
        (ex/raise :type :not-found
                  :table table
                  :hint "database object not found"))
@@ -267,13 +301,28 @@
   (instance? PGpoint v))
 
 (defn pgarray?
-  [v]
-  (instance? PgArray v))
+  ([v] (instance? PgArray v))
+  ([v type]
+   (and (instance? PgArray v)
+        (= type (.getBaseTypeName ^PgArray v)))))
 
 (defn pgarray-of-uuid?
   [v]
   (and (pgarray? v) (= "uuid" (.getBaseTypeName ^PgArray v))))
 
+(defn decode-pgarray
+  ([v] (into [] (.getArray ^PgArray v)))
+  ([v in] (into in (.getArray ^PgArray v)))
+  ([v in xf] (into in xf (.getArray ^PgArray v))))
+
+(defn pgarray->set
+  [v]
+  (set (.getArray ^PgArray v)))
+
+(defn pgarray->vector
+  [v]
+  (vec (.getArray ^PgArray v)))
+
 (defn pgpoint
   [p]
   (PGpoint. (:x p) (:y p)))
@@ -285,7 +334,6 @@
       (.createArrayOf conn ^String type (into-array Object objects))
       (.createArrayOf conn ^String type objects))))
 
-
 (defn decode-pgpoint
   [^PGpoint v]
   (gpt/point (.-x v) (.-y v)))
@@ -331,7 +379,7 @@
         val (.getValue o)]
     (if (or (= typ "json")
             (= typ "jsonb"))
-      (json/decode-str val)
+      (json/read val)
       val)))
 
 (defn decode-transit-pgobject
@@ -367,16 +415,7 @@
   [data]
   (doto (org.postgresql.util.PGobject.)
     (.setType "jsonb")
-    (.setValue (json/encode-str data))))
-
-(defn pgarray->set
-  [v]
-  (set (.getArray ^PgArray v)))
-
-(defn pgarray->vector
-  [v]
-  (vec (.getArray ^PgArray v)))
-
+    (.setValue (json/write-str data))))
 
 ;; --- Locks
 

backend/src/app/emails.clj

@@ -7,12 +7,12 @@
 (ns app.emails
   "Main api for send emails."
   (:require
+   [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.util.emails :as emails]
-   [app.util.logging :as l]
    [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
@@ -54,10 +54,10 @@
 (defn allow-send-emails?
   [conn profile]
   (when-not (:is-muted profile false)
-    (let [complaint-threshold (cfg/get :profile-complaint-threshold)
-          complaint-max-age   (cfg/get :profile-complaint-max-age)
-          bounce-threshold    (cfg/get :profile-bounce-threshold)
-          bounce-max-age      (cfg/get :profile-bounce-max-age)
+    (let [complaint-threshold (cf/get :profile-complaint-threshold)
+          complaint-max-age   (cf/get :profile-complaint-max-age)
+          bounce-threshold    (cf/get :profile-bounce-threshold)
+          bounce-max-age      (cf/get :profile-bounce-max-age)
 
           {:keys [complaints bounces] :as result}
           (db/exec-one! conn [sql:profile-complaint-report
@@ -66,8 +66,8 @@
                               (:id profile)
                               (db/interval bounce-max-age)])]
 
-      (and (< complaints complaint-threshold)
-           (< bounces bounce-threshold)))))
+      (and (< (or complaints 0) complaint-threshold)
+           (< (or bounces 0) bounce-threshold)))))
 
 (defn has-complaint-reports?
   ([conn email] (has-complaint-reports? conn email nil))
@@ -140,19 +140,17 @@
 
 (declare send-console!)
 
-(s/def ::username ::cfg/smtp-username)
-(s/def ::password ::cfg/smtp-password)
-(s/def ::tls ::cfg/smtp-tls)
-(s/def ::ssl ::cfg/smtp-ssl)
-(s/def ::host ::cfg/smtp-host)
-(s/def ::port ::cfg/smtp-port)
-(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
-(s/def ::default-from ::cfg/smtp-default-from)
-(s/def ::enabled ::cfg/smtp-enabled)
+(s/def ::username ::cf/smtp-username)
+(s/def ::password ::cf/smtp-password)
+(s/def ::tls ::cf/smtp-tls)
+(s/def ::ssl ::cf/smtp-ssl)
+(s/def ::host ::cf/smtp-host)
+(s/def ::port ::cf/smtp-port)
+(s/def ::default-reply-to ::cf/smtp-default-reply-to)
+(s/def ::default-from ::cf/smtp-default-from)
 
 (defmethod ig/pre-init-spec ::sendmail-handler [_]
-  (s/keys :req-un [::enabled]
-          :opt-un [::username
+  (s/keys :opt-un [::username
                    ::password
                    ::tls
                    ::ssl
@@ -164,9 +162,12 @@
 (defmethod ig/init-key ::sendmail-handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (if (:enabled cfg)
-      (emails/send! cfg props)
-      (send-console! cfg props))))
+    (let [enabled? (or (contains? cf/flags :smtp)
+                       (cf/get :smtp-enabled)
+                       (:enabled task))]
+      (if enabled?
+        (emails/send! cfg props)
+        (send-console! cfg props)))))
 
 (defn- send-console!
   [cfg email]

backend/src/app/http.clj

@@ -8,104 +8,101 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
+   [app.config :as cf]
+   [app.http.doc :as doc]
    [app.http.errors :as errors]
    [app.http.middleware :as middleware]
    [app.metrics :as mtx]
-   [app.util.logging :as l]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [reitit.ring :as rr]
-   [ring.adapter.jetty9 :as jetty])
+   [yetti.adapter :as yt])
   (:import
    org.eclipse.jetty.server.Server
-   org.eclipse.jetty.server.handler.ErrorHandler
    org.eclipse.jetty.server.handler.StatisticsHandler))
 
-(declare router-handler)
+(declare wrap-router)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP SERVER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::handler fn?)
 (s/def ::router some?)
-(s/def ::ws (s/map-of ::us/string fn?))
 (s/def ::port ::us/integer)
+(s/def ::host ::us/string)
 (s/def ::name ::us/string)
-
-(defmethod ig/pre-init-spec ::server [_]
-  (s/keys :req-un [::port]
-          :opt-un [::ws ::name ::mtx/metrics ::router ::handler]))
+(s/def ::max-threads ::cf/http-server-max-threads)
+(s/def ::min-threads ::cf/http-server-min-threads)
 
 (defmethod ig/prep-key ::server
   [_ cfg]
-  (merge {:name "http"} (d/without-nils cfg)))
+  (merge {:name "http"
+          :min-threads 4
+          :max-threads 60
+          :port 6060
+          :host "0.0.0.0"}
+         (d/without-nils cfg)))
+
+(defmethod ig/pre-init-spec ::server [_]
+  (s/keys :req-un [::port ::host ::name ::min-threads ::max-threads]
+          :opt-un [::mtx/metrics ::router ::handler]))
+
+(defn- instrument-metrics
+  [^Server server metrics]
+  (let [stats (doto (StatisticsHandler.)
+                (.setHandler (.getHandler server)))]
+    (.setHandler server stats)
+    (mtx/instrument-jetty! (:registry metrics) stats)
+    server))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [handler router ws port name metrics] :as opts}]
-  (l/info :msg "starting http server" :port port :name name)
-  (let [pre-start (fn [^Server server]
-                    (let [handler (doto (ErrorHandler.)
-                                    (.setShowStacks true)
-                                    (.setServer server))]
-                      (.setErrorHandler server ^ErrorHandler handler)
-                      (when metrics
-                        (let [stats (StatisticsHandler.)]
-                          (.setHandler ^StatisticsHandler stats (.getHandler server))
-                          (.setHandler server stats)
-                          (mtx/instrument-jetty! (:registry metrics) stats)))))
-
-        options   (merge
-                   {:port port
-                    :h2c? true
-                    :join? false
-                    :allow-null-path-info true
-                    :configurator pre-start}
-                   (when (seq ws)
-                     {:websockets ws}))
-
-        handler   (cond
-                    (fn? handler)  handler
-                    (some? router) (router-handler router)
-                    :else (ex/raise :type :internal
-                                    :code :invalid-argument
-                                    :hint "Missing `handler` or `router` option."))
-
-        server    (jetty/run-jetty handler options)]
-    (assoc opts :server server)))
+  [_ {:keys [handler router port name metrics host] :as opts}]
+  (l/info :hint "starting http server"
+          :port port :host host :name name
+          :min-threads (:min-threads opts)
+          :max-threads (:max-threads opts))
+  (let [options {:http/port port
+                 :http/host host
+                 :thread-pool/max-threads (:max-threads opts)
+                 :thread-pool/min-threads (:min-threads opts)
+                 :ring/async true}
+        handler (cond
+                  (fn? handler)  handler
+                  (some? router) (wrap-router router)
+                  :else (ex/raise :type :internal
+                                  :code :invalid-argument
+                                  :hint "Missing `handler` or `router` option."))
+        server  (-> (yt/server handler (d/without-nils options))
+                    (cond-> metrics (instrument-metrics metrics)))]
+    (assoc opts :server (yt/start! server))))
 
 (defmethod ig/halt-key! ::server
   [_ {:keys [server name port] :as opts}]
-  (l/info :msg "stoping http server"
-          :name name
-          :port port)
-  (jetty/stop-server server))
+  (l/info :msg "stoping http server" :name name :port port)
+  (yt/stop! server))
 
-(defn- router-handler
+(defn- wrap-router
   [router]
-  (let [handler (rr/ring-handler router
-                                 (rr/routes
-                                  (rr/create-resource-handler {:path "/"})
-                                  (rr/create-default-handler))
-                                 {:middleware [middleware/server-timing]})]
-    (fn [request]
-      (try
-        (handler request)
-        (catch Throwable e
-          (try
-            (let [cdata (errors/get-error-context request e)]
-              (l/update-thread-context! cdata)
-              (l/error :hint "unhandled exception"
-                       :message (ex-message e)
-                       :error-id (str (:id cdata))
-                       :cause e))
-            {:status 500 :body "internal server error"}
-            (catch Throwable e
-              (l/error :hint "unhandled exception"
-                       :message (ex-message e)
-                       :cause e)
-              {:status 500 :body "internal server error"})))))))
-
+  (let [default (rr/routes
+                 (rr/create-resource-handler {:path "/"})
+                 (rr/create-default-handler))
+        options {:middleware [middleware/wrap-server-timing]
+                 :inject-match? false
+                 :inject-router? false}
+        handler (rr/ring-handler router default options)]
+    (fn [request respond _]
+      (handler request respond (fn [cause]
+                                 (l/error :hint "unexpected error processing request"
+                                          ::l/context (errors/get-error-context request cause)
+                                          :query-string (:query-string request)
+                                          :cause cause)
+                                 (respond {:status 500 :body "internal server error"}))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Main Handler (Router)
+;; HTTP ROUTER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::rpc map?)
@@ -114,12 +111,17 @@
 (s/def ::storage map?)
 (s/def ::assets map?)
 (s/def ::feedback fn?)
+(s/def ::ws fn?)
+(s/def ::audit-http-handler fn?)
+(s/def ::debug map?)
 
 (defmethod ig/pre-init-spec ::router [_]
-  (s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
+  (s/keys :req-un [::rpc ::session ::mtx/metrics ::ws
+                   ::oauth ::storage ::assets ::feedback
+                   ::debug ::audit-http-handler]))
 
 (defmethod ig/init-key ::router
-  [_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
+  [_ {:keys [ws session rpc oauth metrics assets feedback debug] :as cfg}]
   (rr/router
    [["/metrics" {:get (:handler metrics)}]
     ["/assets" {:middleware [[middleware/format-response-body]
@@ -130,29 +132,52 @@
      ["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
      ["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
 
-    ["/dbg"
-     ["/error-by-id/:id" {:get (:error-report-handler cfg)}]]
+    ["/dbg" {:middleware [[middleware/multipart-params]
+                          [middleware/params]
+                          [middleware/keyword-params]
+                          [middleware/format-response-body]
+                          [middleware/errors errors/handle]
+                          [middleware/cookies]
+                          [(:middleware session)]]}
+     ["" {:get (:index debug)}]
+     ["/error-by-id/:id" {:get (:retrieve-error debug)}]
+     ["/error/:id" {:get (:retrieve-error debug)}]
+     ["/error" {:get (:retrieve-error-list debug)}]
+     ["/file/data" {:get (:retrieve-file-data debug)
+                    :post (:upload-file-data debug)}]
+     ["/file/changes" {:get (:retrieve-file-changes debug)}]]
 
     ["/webhooks"
      ["/sns" {:post (:sns-webhook cfg)}]]
 
-    ["/api" {:middleware [
-                          ;; Temporary disabled
-                          #_[middleware/etag]
-                          [middleware/format-response-body]
+    ["/ws/notifications"
+     {:middleware [[middleware/params]
+                   [middleware/keyword-params]
+                   [middleware/format-response-body]
+                   [middleware/errors errors/handle]
+                   [middleware/cookies]
+                   [(:middleware session)]]
+      :get ws}]
+
+    ["/api" {:middleware [[middleware/cors]
                           [middleware/params]
                           [middleware/multipart-params]
                           [middleware/keyword-params]
+                          [middleware/format-response-body]
                           [middleware/parse-request-body]
                           [middleware/errors errors/handle]
                           [middleware/cookies]]}
 
+     ["/health" {:get (:health-check debug)}]
+     ["/_doc" {:get (doc/handler rpc)}]
      ["/feedback" {:middleware [(:middleware session)]
                    :post feedback}]
-
      ["/auth/oauth/:provider" {:post (:handler oauth)}]
      ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
 
+     ["/audit/events" {:middleware [(:middleware session)]
+                       :post (:audit-http-handler cfg)}]
+
      ["/rpc" {:middleware [(:middleware session)]}
       ["/query/:type" {:get (:query-handler rpc)
                        :post (:query-handler rpc)}]

backend/src/app/http/assets.clj

@@ -13,9 +13,12 @@
    [app.db :as db]
    [app.metrics :as mtx]
    [app.storage :as sto]
+   [app.util.async :as async]
    [app.util.time :as dt]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]))
 
 (def ^:private cache-max-age
   (dt/duration {:hours 24}))
@@ -52,10 +55,10 @@
        :body (sto/get-object-bytes storage obj)}
 
       :s3
-      (let [url (sto/get-object-url storage obj {:max-age signature-max-age})]
+      (let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
         {:status 307
          :headers {"location" (str url)
-                   "x-host"   (:host url)
+                   "x-host"   (cond-> host port (str ":" port))
                    "cache-control" (str "max-age=" (inst-ms cache-max-age))}
          :body ""})
 
@@ -69,29 +72,38 @@
          :body ""}))))
 
 (defn- generic-handler
-  [{:keys [storage] :as cfg} _request id]
-  (let [obj (sto/get-object storage id)]
-    (if obj
-      (serve-object cfg obj)
-      {:status 404 :body ""})))
+  [{:keys [storage executor] :as cfg} request kf]
+  (async/with-dispatch executor
+    (let [id   (get-in request [:path-params :id])
+          mobj (get-file-media-object storage id)
+          obj  (sto/get-object storage (kf mobj))]
+      (if obj
+        (serve-object cfg obj)
+        {:status 404 :body ""}))))
 
 (defn objects-handler
-  [cfg request]
-  (let [id (get-in request [:path-params :id])]
-    (generic-handler cfg request (coerce-id id))))
+  [{:keys [storage executor] :as cfg} request respond raise]
+  (-> (async/with-dispatch executor
+        (let [id (get-in request [:path-params :id])
+              id (coerce-id id)
+              obj  (sto/get-object storage id)]
+          (if obj
+            (serve-object cfg obj)
+            {:status 404 :body ""})))
+      (p/then respond)
+      (p/catch raise)))
 
 (defn file-objects-handler
-  [{:keys [storage] :as cfg} request]
-  (let [id   (get-in request [:path-params :id])
-        mobj (get-file-media-object storage id)]
-    (generic-handler cfg request (:media-id mobj))))
+  [cfg request respond raise]
+  (-> (generic-handler cfg request :media-id)
+      (p/then respond)
+      (p/catch raise)))
 
 (defn file-thumbnails-handler
-  [{:keys [storage] :as cfg} request]
-  (let [id   (get-in request [:path-params :id])
-        mobj (get-file-media-object storage id)]
-    (generic-handler cfg request (or (:thumbnail-id mobj) (:media-id mobj)))))
-
+  [cfg request respond raise]
+  (-> (generic-handler cfg request #(or (:thumbnail-id %) (:media-id %)))
+      (p/then respond)
+      (p/catch raise)))
 
 ;; --- Initialization
 
@@ -101,10 +113,16 @@
 (s/def ::signature-max-age ::dt/duration)
 
 (defmethod ig/pre-init-spec ::handlers [_]
-  (s/keys :req-un [::storage ::mtx/metrics ::assets-path ::cache-max-age ::signature-max-age]))
+  (s/keys :req-un [::storage
+                   ::wrk/executor
+                   ::mtx/metrics
+                   ::assets-path
+                   ::cache-max-age
+                   ::signature-max-age]))
 
 (defmethod ig/init-key ::handlers
   [_ cfg]
-  {:objects-handler #(objects-handler cfg %)
-   :file-objects-handler #(file-objects-handler cfg %)
-   :file-thumbnails-handler #(file-thumbnails-handler cfg %)})
+  {:objects-handler (partial objects-handler cfg)
+   :file-objects-handler (partial file-objects-handler cfg)
+   :file-thumbnails-handler (partial file-thumbnails-handler cfg)})
+

backend/src/app/http/awsns.clj

@@ -8,10 +8,10 @@
   "AWS SNS webhook handler for bounces."
   (:require
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.util.http :as http]
-   [app.util.logging :as l]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
@@ -26,25 +26,30 @@
 
 (defmethod ig/init-key ::handler
   [_ cfg]
-  (fn [request]
-    (let [body  (parse-json (slurp (:body request)))
-          mtype (get body "Type")]
-      (cond
-        (= mtype "SubscriptionConfirmation")
-        (let [surl   (get body "SubscribeURL")
-              stopic (get body "TopicArn")]
-          (l/info :action "subscription received" :topic stopic :url surl)
-          (http/send! {:uri surl :method :post :timeout 10000}))
+  (fn [request respond _]
+    (try
+      (let [body  (parse-json (slurp (:body request)))
+            mtype (get body "Type")]
+        (cond
+          (= mtype "SubscriptionConfirmation")
+          (let [surl   (get body "SubscribeURL")
+                stopic (get body "TopicArn")]
+            (l/info :action "subscription received" :topic stopic :url surl)
+            (http/send! {:uri surl :method :post :timeout 10000}))
 
-        (= mtype "Notification")
-        (when-let [message (parse-json (get body "Message"))]
-          (let [notification (parse-notification cfg message)]
-            (process-report cfg notification)))
+          (= mtype "Notification")
+          (when-let [message (parse-json (get body "Message"))]
+            (let [notification (parse-notification cfg message)]
+              (process-report cfg notification)))
 
-        :else
-        (l/warn :hint "unexpected data received"
-                :report (pr-str body)))
-      {:status 200 :body ""})))
+          :else
+          (l/warn :hint "unexpected data received"
+                  :report (pr-str body))))
+      (catch Throwable cause
+        (l/error :hint "unexpected exception on awsns handler"
+                 :cause cause)))
+
+    (respond {:status 200 :body ""})))
 
 (defn- parse-bounce
   [data]
@@ -173,14 +178,14 @@
 
 (defn- process-report
   [cfg {:keys [type profile-id] :as report}]
-  (l/trace :action "procesing report" :report (pr-str report))
+  (l/trace :action "processing report" :report (pr-str report))
   (cond
     ;; In this case we receive a bounce/complaint notification without
     ;; confirmed identity, we just emit a warning but do nothing about
     ;; it because this is not a normal case. All notifications should
     ;; come with profile identity.
     (nil? profile-id)
-    (l/warn :msg "a notification without identity recevied from AWS"
+    (l/warn :msg "a notification without identity received from AWS"
             :report (pr-str report))
 
     (= "bounce" type)

backend/src/app/http/debug.clj

@@ -0,0 +1,227 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.debug
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.rpc.mutations.files :as m.files]
+   [app.rpc.queries.profile :as profile]
+   [app.util.async :as async]
+   [app.util.blob :as blob]
+   [app.util.template :as tmpl]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [datoteka.core :as fs]
+   [fipp.edn :as fpp]
+   [integrant.core :as ig]
+   [promesa.core :as p]))
+
+;; (selmer.parser/cache-off!)
+
+(defn authorized?
+  [pool {:keys [profile-id]}]
+  (or (= "devenv" (cf/get :host))
+      (let [profile (ex/ignoring (profile/retrieve-profile-data pool profile-id))
+            admins  (or (cf/get :admins) #{})]
+        (contains? admins (:email profile)))))
+
+(defn index
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  {:status 200
+   :headers {"content-type" "text/html"}
+   :body (-> (io/resource "templates/debug.tmpl")
+             (tmpl/render {}))})
+
+
+(def sql:retrieve-range-of-changes
+  "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
+
+(def sql:retrieve-single-change
+  "select revn, changes, data from file_change where file_id=? and revn = ?")
+
+(defn prepare-response
+  [{:keys [params] :as request} body]
+  (when-not body
+    (ex/raise :type :not-found
+              :code :enpty-data
+              :hint "empty response"))
+
+  (cond-> {:status 200
+           :headers {"content-type" "application/transit+json"}
+           :body body}
+    (contains? params :download)
+    (update :headers assoc "content-disposition" "attachment")))
+
+(defn retrieve-file-data
+  [{:keys [pool]} {:keys [params] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (let [file-id (some-> (get-in request [:params :file-id]) uuid/uuid)
+        revn    (some-> (get-in request [:params :revn]) d/parse-integer)]
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments))
+
+    (let [data (if (integer? revn)
+                 (some-> (db/exec-one! pool [sql:retrieve-single-change file-id revn]) :data)
+                 (some-> (db/get-by-id pool :file file-id) :data))]
+      (if (contains? params :download)
+        (-> (prepare-response request data)
+            (update :headers assoc "content-type" "application/octet-stream"))
+        (prepare-response request (some-> data blob/decode))))))
+
+(defn upload-file-data
+  [{:keys [pool]} {:keys [profile-id params] :as request}]
+  (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
+        data       (some-> params :file :tempfile fs/slurp-bytes blob/decode)]
+
+    (if (and data project-id)
+      (let [fname (str "imported-file-" (dt/now))]
+        (m.files/create-file pool {:id (uuid/next)
+                                   :name fname
+                                   :project-id project-id
+                                   :profile-id profile-id
+                                   :data data})
+        {:status 200
+         :body "OK"})
+      {:status 500
+       :body "error"})))
+
+(defn retrieve-file-changes
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (let [file-id (some-> (get-in request [:params :id]) uuid/uuid)
+        revn    (or (get-in request [:params :revn]) "latest")]
+
+    (when (or (not file-id) (not revn))
+      (ex/raise :type :validation
+                :code :invalid-arguments
+                :hint "missing arguments"))
+
+    (cond
+      (d/num-string? revn)
+      (let [item (db/exec-one! pool [sql:retrieve-single-change file-id (d/parse-integer revn)])]
+        (prepare-response request (some-> item :changes blob/decode vec)))
+
+      (str/includes? revn ":")
+      (let [[start end] (->> (str/split revn #":")
+                             (map str/trim)
+                             (map d/parse-integer))
+            items       (db/exec! pool [sql:retrieve-range-of-changes file-id start end])]
+        (prepare-response request
+                          (some->> items
+                                   (map :changes)
+                                   (map blob/decode)
+                                   (mapcat identity)
+                                   (vec))))
+      :else
+      (ex/raise :type :validation :code :invalid-arguments))))
+
+
+(defn retrieve-error
+  [{:keys [pool]} request]
+  (letfn [(parse-id [request]
+            (let [id (get-in request [:path-params :id])
+                  id (us/uuid-conformer id)]
+              (when (uuid? id)
+                id)))
+
+          (retrieve-report [id]
+            (ex/ignoring
+             (some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject)))
+
+          (render-template [report]
+            (let [context (dissoc report
+                                  :trace :cause :params :data :spec-problems
+                                  :spec-explain :spec-value :error :explain :hint)
+                  params  {:context (with-out-str (fpp/pprint context {:width 300}))
+                           :hint    (:hint report)
+                           :spec-explain  (:spec-explain report)
+                           :spec-problems (:spec-problems report)
+                           :spec-value    (:spec-value report)
+                           :data          (:data report)
+                           :trace         (or (:trace report)
+                                              (some-> report :error :trace))
+                           :params        (:params report)}]
+              (-> (io/resource "templates/error-report.tmpl")
+                  (tmpl/render params))))
+          ]
+
+    (when-not (authorized? pool request)
+      (ex/raise :type :authentication
+                :code :only-admins-allowed))
+
+    (let [result (some-> (parse-id request)
+                         (retrieve-report)
+                         (render-template))]
+      (if result
+        {:status 200
+         :headers {"content-type" "text/html; charset=utf-8"
+                   "x-robots-tag" "noindex"}
+         :body result}
+        {:status 404
+         :body "not found"}))))
+
+(def sql:error-reports
+  "select id, created_at from server_error_report order by created_at desc limit 100")
+
+(defn retrieve-error-list
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+  (let [items (db/exec! pool [sql:error-reports])
+        items (map #(update % :created-at dt/format-instant :rfc1123) items)]
+    {:status 200
+     :headers {"content-type" "text/html; charset=utf-8"
+               "x-robots-tag" "noindex"}
+     :body (-> (io/resource "templates/error-list.tmpl")
+               (tmpl/render {:items items}))}))
+
+(defn health-check
+  "Mainly a task that performs a health check."
+  [{:keys [pool]} _]
+  (db/with-atomic [conn pool]
+    (db/exec-one! conn ["select count(*) as count from server_prop;"])
+    {:status 200 :body "Ok"}))
+
+(defn- wrap-async
+  [{:keys [executor] :as cfg} f]
+  (fn [request respond raise]
+    (-> (async/with-dispatch executor
+          (f cfg request))
+        (p/then respond)
+        (p/catch raise))))
+
+(defmethod ig/pre-init-spec ::handlers [_]
+  (s/keys :req-un [::db/pool ::wrk/executor]))
+
+(defmethod ig/init-key ::handlers
+  [_ cfg]
+  {:index (wrap-async cfg index)
+   :health-check (wrap-async cfg health-check)
+   :retrieve-file-data (wrap-async cfg retrieve-file-data)
+   :retrieve-file-changes (wrap-async cfg retrieve-file-changes)
+   :retrieve-error (wrap-async cfg retrieve-error)
+   :retrieve-error-list (wrap-async cfg retrieve-error-list)
+   :upload-file-data (wrap-async cfg upload-file-data)})

backend/src/app/http/doc.clj

@@ -0,0 +1,54 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.doc
+  "API autogenerated documentation."
+  (:require
+   [app.common.data :as d]
+   [app.config :as cf]
+   [app.util.services :as sv]
+   [app.util.template :as tmpl]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [pretty-spec.core :as ps]))
+
+(defn get-spec-str
+  [k]
+  (with-out-str
+    (ps/pprint (s/form k)
+               {:ns-aliases {"clojure.spec.alpha" "s"
+                             "clojure.core.specs.alpha" "score"
+                             "clojure.core" nil}})))
+
+(defn prepare-context
+  [rpc]
+  (letfn [(gen-doc [type [name f]]
+            (let [mdata (meta f)]
+              ;; (prn name mdata)
+              {:type (d/name type)
+               :name (d/name name)
+               :auth (:auth mdata true)
+               :docs (::sv/docs mdata)
+               :spec (get-spec-str (::sv/spec mdata))}))]
+    {:query-methods
+     (into []
+           (map (partial gen-doc :query))
+           (->> rpc :methods :query (sort-by first)))
+     :mutation-methods
+     (into []
+           (map (partial gen-doc :mutation))
+           (->> rpc :methods :mutation (sort-by first)))}))
+
+(defn handler
+  [rpc]
+  (let [context (prepare-context rpc)]
+    (if (contains? cf/flags :backend-api-doc)
+      (fn [_ respond _]
+        (respond {:status 200
+                  :body (-> (io/resource "api-doc.tmpl")
+                            (tmpl/render context))}))
+      (fn [_ respond _]
+        (respond {:status 404 :body ""})))))

backend/src/app/http/errors.clj

@@ -8,30 +8,38 @@
   "A errors handling for the http server."
   (:require
    [app.common.exceptions :as ex]
-   [app.common.uuid :as uuid]
-   [app.util.logging :as l]
-   [cuerdas.core :as str]
-   [expound.alpha :as expound]))
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]))
 
-(defn- explain-error
-  [error]
-  (with-out-str
-    (expound/printer (:data error))))
+(defn- parse-client-ip
+  [{:keys [headers] :as request}]
+  (or (some-> (get headers "x-forwarded-for") (str/split ",") first)
+      (get headers "x-real-ip")
+      (get request :remote-addr)))
 
 (defn get-error-context
   [request error]
-  (let [edata (ex-data error)]
+  (let [data (ex-data error)]
     (merge
-     {:id      (uuid/next)
-      :path    (:uri request)
-      :method  (:request-method request)
-      :params  (:params request)
-      :data    edata}
+     {:path          (:uri request)
+      :method        (:request-method request)
+      :hint          (ex-message error)
+      :params        (:params request)
+
+      :spec-problems (some->> data ::s/problems (take 10) seq vec)
+      :spec-value    (some->> data ::s/value)
+      :data          (some-> data (dissoc ::s/problems ::s/value ::s/spec))
+      :ip-addr       (parse-client-ip request)
+      :profile-id    (:profile-id request)}
+
      (let [headers (:headers request)]
        {:user-agent (get headers "user-agent")
         :frontend-version (get headers "x-frontend-version" "unknown")})
-     (when (and (map? edata) (:data edata))
-       {:explain (explain-error edata)}))))
+
+     (when (and data (::s/problems data))
+       {:spec-explain (us/pretty-explain data)}))))
 
 (defmulti handle-exception
   (fn [err & _rest]
@@ -43,43 +51,31 @@
   [err _]
   {:status 401 :body (ex-data err)})
 
-
 (defmethod handle-exception :restriction
   [err _]
   {:status 400 :body (ex-data err)})
 
 (defmethod handle-exception :validation
-  [err req]
-  (let [header (get-in req [:headers "accept"])
-        edata  (ex-data err)]
-    (if (and (= :spec-validation (:code edata))
-             (str/starts-with? header "text/html"))
-      {:status 400
-       :headers {"content-type" "text/html"}
-       :body (str "<pre style='font-size:16px'>"
-                  (explain-error edata)
-                  "</pre>\n")}
-      {:status 400
-       :body   (cond-> edata
-                 (map? (:data edata))
-                 (-> (assoc :explain (explain-error edata))
-                     (dissoc :data)))})))
+  [err _]
+  (let [data    (ex-data err)
+        explain (us/pretty-explain data)]
+    {:status 400
+     :body (-> data
+               (dissoc ::s/problems)
+               (dissoc ::s/value)
+               (cond-> explain (assoc :explain explain)))}))
 
 (defmethod handle-exception :assertion
   [error request]
-  (let [edata (ex-data error)
-        cdata (get-error-context request error)]
-    (l/update-thread-context! cdata)
-    (l/error :hint "internal error: assertion"
-             :error-id (str (:id cdata))
+  (let [edata (ex-data error)]
+    (l/error ::l/raw (ex-message error)
+             ::l/context (get-error-context request error)
              :cause error)
 
     {:status 500
      :body {:type :server-error
             :code :assertion
-            :data (-> edata
-                      (assoc :explain (explain-error edata))
-                      (dissoc :data))}}))
+            :data (dissoc edata ::s/problems ::s/value ::s/spec)}}))
 
 (defmethod handle-exception :not-found
   [err _]
@@ -96,11 +92,9 @@
     (if (and (ex/exception? (:rollback edata))
              (ex/exception? (:handling edata)))
       (handle-exception (:handling edata) request)
-      (let [cdata (get-error-context request error)]
-        (l/update-thread-context! cdata)
-        (l/error :hint "internal error"
-                 :error-message (ex-message error)
-                 :error-id (str (:id cdata))
+      (do
+        (l/error ::l/raw (ex-message error)
+                 ::l/context (get-error-context request error)
                  :cause error)
         {:status 500
          :body {:type :server-error
@@ -110,16 +104,10 @@
 
 (defmethod handle-exception org.postgresql.util.PSQLException
   [error request]
-  (let [cdata (get-error-context request error)
-        state (.getSQLState ^java.sql.SQLException error)]
-
-    (l/update-thread-context! cdata)
-    (l/error :hint "psql exception"
-             :error-message (ex-message error)
-             :error-id (str (:id cdata))
-             :sql-state state
+  (let [state (.getSQLState ^java.sql.SQLException error)]
+    (l/error ::l/raw (ex-message error)
+             ::l/context (get-error-context request error)
              :cause error)
-
     (cond
       (= state "57014")
       {:status 504

backend/src/app/http/feedback.clj

@@ -10,57 +10,65 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.emails :as eml]
    [app.rpc.queries.profile :as profile]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]))
 
-(declare send-feedback)
+(declare ^:private send-feedback)
+(declare ^:private handler)
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
+  (s/keys :req-un [::db/pool ::wrk/executor]))
 
 (defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as scfg}]
-  (let [ftoken   (cfg/get :feedback-token ::no-token)
-        enabled  (cfg/get :feedback-enabled)]
-    (fn [{:keys [profile-id] :as request}]
-      (let [token  (get-in request [:headers "x-feedback-token"])
-            params (d/merge (:params request)
-                            (:body-params request))]
+  [_ {:keys [executor] :as cfg}]
+  (let [enabled? (contains? cf/flags :user-feedback)]
+    (if enabled?
+      (fn [request respond raise]
+        (-> (px/submit! executor #(handler cfg request))
+            (p/then' respond)
+            (p/catch raise)))
+      (fn [_ _ raise]
+        (raise (ex/error :type :validation
+                         :code :feedback-disabled
+                         :hint "feedback module is disabled"))))))
 
-        (when-not enabled
-          (ex/raise :type :validation
-                    :code :feedback-disabled
-                    :hint "feedback module is disabled"))
+(defn- handler
+  [{:keys [pool] :as cfg} {:keys [profile-id] :as request}]
+  (let [ftoken (cf/get :feedback-token ::no-token)
+        token  (get-in request [:headers "x-feedback-token"])
+        params (d/merge (:params request)
+                        (:body-params request))]
+    (cond
+      (uuid? profile-id)
+      (let [profile (profile/retrieve-profile-data pool profile-id)
+            params  (assoc params :from (:email profile))]
+        (send-feedback pool profile params))
 
-        (cond
-          (uuid? profile-id)
-          (let [profile (profile/retrieve-profile-data pool profile-id)
-                params  (assoc params :from (:email profile))]
-            (when-not (:is-muted profile)
-              (send-feedback pool profile params)))
+      (= token ftoken)
+      (send-feedback cfg nil params))
 
-          (= token ftoken)
-          (send-feedback scfg nil params))
-
-        {:status 204 :body ""}))))
+    {:status 204 :body ""}))
 
 (s/def ::content ::us/string)
 (s/def ::from    ::us/email)
 (s/def ::subject ::us/string)
-
 (s/def ::feedback
   (s/keys :req-un [::from ::subject ::content]))
 
-(defn send-feedback
+(defn- send-feedback
   [pool profile params]
   (let [params      (us/conform ::feedback params)
-        destination (cfg/get :feedback-destination)]
+        destination (cf/get :feedback-destination)]
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/feedback
+                :from     destination
                 :to       destination
                 :profile  profile
                 :reply-to (:from params)

backend/src/app/http/middleware.clj

@@ -6,28 +6,28 @@
 
 (ns app.http.middleware
   (:require
+   [app.common.logging :as l]
    [app.common.transit :as t]
-   [app.metrics :as mtx]
+   [app.config :as cf]
    [app.util.json :as json]
-   [app.util.logging :as l]
-   [buddy.core.codecs :as bc]
-   [buddy.core.hash :as bh]
-   [clojure.java.io :as io]
    [ring.core.protocols :as rp]
    [ring.middleware.cookies :refer [wrap-cookies]]
    [ring.middleware.keyword-params :refer [wrap-keyword-params]]
    [ring.middleware.multipart-params :refer [wrap-multipart-params]]
-   [ring.middleware.params :refer [wrap-params]]))
+   [ring.middleware.params :refer [wrap-params]]
+   [yetti.adapter :as yt]))
 
 (defn wrap-server-timing
   [handler]
-  (let [seconds-from #(float (/ (- (System/nanoTime) %) 1000000000))]
-    (fn [request]
-      (let [start    (System/nanoTime)
-            response (handler request)]
-        (update response :headers
-                (fn [headers]
-                  (assoc headers "Server-Timing" (str "total;dur=" (seconds-from start)))))))))
+  (letfn [(get-age [start]
+            (float (/ (- (System/nanoTime) start) 1000000000)))
+
+          (update-headers [headers start]
+            (assoc headers "Server-Timing" (str "total;dur=" (get-age start))))]
+
+    (fn [request respond raise]
+      (let [start (System/nanoTime)]
+        (handler request #(respond (update % :headers update-headers start)) raise)))))
 
 (defn wrap-parse-request-body
   [handler]
@@ -36,84 +36,103 @@
               (t/read! reader)))
 
           (parse-json [body]
-            (let [reader (io/reader body)]
-              (json/read reader)))
+            (json/read body))
 
-          (parse [type body]
-            (try
-              (case type
-                :json (parse-json body)
-                :transit (parse-transit body))
-              (catch Exception e
-                (let [data {:type :parse
-                            :hint "unable to parse request body"
-                            :message (ex-message e)}]
-                  {:status 400
-                   :headers {"content-type" "application/transit+json"}
-                   :body (t/encode-str data {:type :json-verbose})}))))]
+          (handle-request [{:keys [headers body] :as request}]
+            (let [ctype (get headers "content-type")]
+              (case ctype
+                "application/transit+json"
+                (let [params (parse-transit body)]
+                  (-> request
+                      (assoc :body-params params)
+                      (update :params merge params)))
 
-    (fn [{:keys [headers body] :as request}]
-      (let [ctype (get headers "content-type")]
-        (handler
-         (case ctype
-           "application/transit+json"
-           (let [params (parse :transit body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+                "application/json"
+                (let [params (parse-json body)]
+                  (-> request
+                      (assoc :body-params params)
+                      (update :params merge params)))
 
-           "application/json"
-           (let [params (parse :json body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+                request)))
 
-           request))))))
+          (handle-exception [cause]
+            (let [data {:type :validation
+                        :code :unable-to-parse-request-body
+                        :hint "malformed params"}]
+              (l/error :hint (ex-message cause) :cause cause)
+              {:status 400
+               :headers {"content-type" "application/transit+json"}
+               :body (t/encode-str data {:type :json-verbose})}))]
+
+    (fn [request respond raise]
+      (try
+        (let [request (handle-request request)]
+          (handler request respond raise))
+        (catch Exception cause
+          (respond (handle-exception cause)))))))
 
 (def parse-request-body
   {:name ::parse-request-body
    :compile (constantly wrap-parse-request-body)})
 
-(defn- transit-streamable-body
-  [data opts]
-  (reify rp/StreamableResponseBody
-    (write-body-to-stream [_ response output-stream]
-      (try
-        (let [tw (t/writer output-stream opts)]
-          (t/write! tw data))
-        (finally
-          (.close ^java.io.OutputStream output-stream))))))
+(defn buffered-output-stream
+  "Returns a buffered output stream that ignores flush calls. This is
+  needed because transit-java calls flush very aggresivelly on each
+  object write."
+  [^java.io.OutputStream os ^long chunk-size]
+  (proxy [java.io.BufferedOutputStream] [os (int chunk-size)]
+    ;; Explicitly do not forward flush
+    (flush [])
+    (close []
+      (proxy-super flush)
+      (proxy-super close))))
 
-(defn- impl-format-response-body
-  [response _request]
-  (let [body (:body response)
-        opts {:type :json-verbose}]
-    (cond
-      (coll? body)
-      (-> response
-          (update :headers assoc "content-type" "application/transit+json")
-          (assoc :body (transit-streamable-body body opts)))
+(def ^:const buffer-size (:http/output-buffer-size yt/base-defaults))
 
-      ;; ;; Temporary disabled
-      ;; (-> response
-      ;;     (update :headers assoc "content-type" "application/transit+json")
-      ;;     (assoc :body
-      ;;            (if (= :post (:request-method request))
-      ;;              (transit-streamable-body body opts)
-      ;;              (t/encode body opts))))
-
-      (nil? body)
-      (assoc response :status 204 :body "")
-
-      :else
-      response)))
-
-(defn- wrap-format-response-body
+(defn wrap-format-response-body
   [handler]
-  (fn [request]
-    (let [response (handler request)]
-      (cond-> response
-        (map? response) (impl-format-response-body request)))))
+  (letfn [(transit-streamable-body [data opts]
+            (reify rp/StreamableResponseBody
+              (write-body-to-stream [_ _ output-stream]
+                ;; Use the same buffer as jetty output buffer size
+                (try
+                  (with-open [bos (buffered-output-stream output-stream buffer-size)]
+                    (let [tw (t/writer bos opts)]
+                      (t/write! tw data)))
+                  (catch org.eclipse.jetty.io.EofException _cause
+                    ;; Do nothing, EOF means client closes connection abruptly
+                    nil)
+                  (catch Throwable cause
+                    (l/warn :hint "unexpected error on encoding response"
+                            :cause cause))))))
+
+          (impl-format-response-body [response {:keys [query-params] :as request}]
+            (let [body   (:body response)
+                  opts   {:type (if (contains? query-params "transit_verbose") :json-verbose :json)}]
+              (cond
+                (:ws response)
+                response
+
+                (coll? body)
+                (-> response
+                    (update :headers assoc "content-type" "application/transit+json")
+                    (assoc :body (transit-streamable-body body opts)))
+
+                (nil? body)
+                (assoc response :status 204 :body "")
+
+                :else
+                response)))
+
+          (handle-response [response request]
+            (cond-> response
+              (map? response) (impl-format-response-body request)))]
+
+    (fn [request respond raise]
+      (handler request
+               (fn [response]
+                 (respond (handle-response response request)))
+               raise))))
 
 (def format-response-body
   {:name ::format-response-body
@@ -121,21 +140,14 @@
 
 (defn wrap-errors
   [handler on-error]
-  (fn [request]
-    (try
-      (handler request)
-      (catch Throwable e
-        (on-error e request)))))
+  (fn [request respond _]
+    (handler request respond (fn [cause]
+                               (-> cause (on-error request) respond)))))
 
 (def errors
   {:name ::errors
    :compile (constantly wrap-errors)})
 
-(def metrics
-  {:name ::metrics
-   :wrap (fn [handler]
-           (mtx/wrap-counter handler {:id "http__requests_counter"
-                                      :help "Absolute http requests counter."}))})
 (def cookies
   {:name ::cookies
    :compile (constantly wrap-cookies)})
@@ -156,42 +168,31 @@
   {:name ::server-timing
    :compile (constantly wrap-server-timing)})
 
-(defn wrap-etag
+(defn wrap-cors
   [handler]
-  (letfn [(generate-etag [{:keys [body] :as response}]
-            (str "W/\"" (-> body bh/blake2b-128 bc/bytes->hex) "\""))
-          (get-match [{:keys [headers] :as request}]
-            (get headers "if-none-match"))]
-    (fn [request]
-      (let [response (handler request)]
-        (if (= :get (:request-method request))
-          (let [etag     (generate-etag response)
-                match    (get-match request)
-                response (update response :headers #(assoc % "ETag" etag))]
-            (cond-> response
-              (and (string? match)
-                   (= :get (:request-method request))
-                   (= etag match))
+  (if-not (contains? cf/flags :cors)
+    handler
+    (letfn [(add-cors-headers [response request]
               (-> response
-                  (assoc :body "")
-                  (assoc :status 304))))
-          response)))))
+                  (update
+                   :headers
+                   (fn [headers]
+                     (-> headers
+                         (assoc "access-control-allow-origin" (get-in request [:headers "origin"]))
+                         (assoc "access-control-allow-methods" "GET,POST,DELETE,OPTIONS,PUT,HEAD,PATCH")
+                         (assoc "access-control-allow-credentials" "true")
+                         (assoc "access-control-expose-headers" "x-requested-with, content-type, cookie")
+                         (assoc "access-control-allow-headers" "x-frontend-version, content-type, accept, x-requested-width"))))))]
+      (fn [request respond raise]
+        (if (= (:request-method request) :options)
+          (-> {:status 200 :body ""}
+              (add-cors-headers request)
+              (respond))
+          (handler request
+                   (fn [response]
+                     (respond (add-cors-headers response request)))
+                   raise))))))
 
-(def etag
-  {:name ::etag
-   :compile (constantly wrap-etag)})
-
-(defn activity-logger
-  [handler]
-  (let [logger "penpot.profile-activity"]
-    (fn [{:keys [headers] :as request}]
-      (let [ip-addr    (get headers "x-forwarded-for")
-            profile-id (:profile-id request)
-            qstring    (:query-string request)]
-        (l/info ::l/async true
-                ::l/logger logger
-                :ip-addr ip-addr
-                :profile-id profile-id
-                :uri (str (:uri request) (when qstring (str "?" qstring)))
-                :method (name (:request-method request)))
-        (handler request)))))
+(def cors
+  {:name ::cors
+   :compile (constantly wrap-cors)})

backend/src/app/http/oauth.clj

@@ -8,6 +8,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.uri :as u]
    [app.config :as cf]
@@ -15,13 +16,15 @@
    [app.loggers.audit :as audit]
    [app.rpc.queries.profile :as profile]
    [app.util.http :as http]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.data.json :as json]
    [clojure.set :as set]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.exec :as px]))
+
+;; TODO: make it fully async (?)
 
 (defn- build-redirect-uri
   [{:keys [provider] :as cfg}]
@@ -58,10 +61,16 @@
           {:token (get data "access_token")
            :type  (get data "token_type")})))
     (catch Exception e
-      (l/error :hint "unexpected error on retrieve-access-token"
-               :cause e)
+      (l/warn :hint "unexpected error on retrieve-access-token" :cause e)
       nil)))
 
+(defn- qualify-props
+  [provider props]
+  (reduce-kv (fn [result k v]
+               (assoc result (keyword (:name provider) (name k)) v))
+             {}
+             props))
+
 (defn- retrieve-user-info
   [{:keys [provider] :as cfg} tdata]
   (try
@@ -76,11 +85,10 @@
           {:backend (:name provider)
            :email (:email info)
            :fullname (:name info)
-           :props (dissoc info :name :email)})))
-
+           :props (->> (dissoc info :name :email)
+                       (qualify-props provider))})))
     (catch Exception e
-      (l/error :hint "unexpected exception on retrieve-user-info"
-               :cause e)
+      (l/warn :hint "unexpected exception on retrieve-user-info" :cause e)
       nil)))
 
 (s/def ::backend ::us/not-empty-string)
@@ -125,7 +133,7 @@
         (when-not (set/subset? provider-roles profile-roles)
           (ex/raise :type :internal
                     :code :unable-to-auth
-                    :hint "not enought permissions"))))
+                    :hint "not enough permissions"))))
 
     (cond-> info
       (some? (:invitation-token state))
@@ -138,15 +146,14 @@
 
 ;; --- HTTP HANDLERS
 
-(defn extract-props
+(defn extract-utm-props
+  "Extracts additional data from user params."
   [params]
   (reduce-kv (fn [params k v]
                (let [sk (name k)]
                  (cond-> params
-                   (or (str/starts-with? sk "pm_")
-                       (str/starts-with? sk "pm-")
-                       (str/starts-with? sk "utm_"))
-                   (assoc (-> sk str/kebab keyword) v))))
+                   (str/starts-with? sk "utm_")
+                   (assoc (->> sk str/kebab (keyword "penpot")) v))))
              {}
              params))
 
@@ -197,6 +204,7 @@
            (sxf request)))
     (let [info   (assoc info
                         :iss :prepared-register
+                        :is-active true
                         :exp (dt/in-future {:hours 48}))
           token  (tokens :generate info)
           params (d/without-nils
@@ -208,28 +216,35 @@
       (redirect-response uri))))
 
 (defn- auth-handler
-  [{:keys [tokens] :as cfg} {:keys [params] :as request}]
-  (let [invitation (:invitation-token params)
-        props      (extract-props params)
-        state      (tokens :generate
-                           {:iss :oauth
-                            :invitation-token invitation
-                            :props props
-                            :exp (dt/in-future "15m")})
-        uri        (build-auth-uri cfg state)]
-    {:status 200
-     :body {:redirect-uri uri}}))
+  [{:keys [tokens executor] :as cfg} {:keys [params] :as request} respond _]
+  (px/run!
+   executor
+   (fn []
+     (let [invitation (:invitation-token params)
+           props      (extract-utm-props params)
+           state      (tokens :generate
+                              {:iss :oauth
+                               :invitation-token invitation
+                               :props props
+                               :exp (dt/in-future "15m")})
+           uri        (build-auth-uri cfg state)]
+
+       (respond
+        {:status 200
+         :body {:redirect-uri uri}})))))
 
 (defn- callback-handler
-  [cfg request]
-  (try
-    (let [info     (retrieve-info cfg request)
-          profile  (retrieve-profile cfg info)]
-      (generate-redirect cfg request info profile))
-    (catch Exception e
-      (l/warn :hint "error on oauth process"
-              :cause e)
-      (generate-error-redirect cfg e))))
+  [{:keys [executor] :as cfg} request respond _]
+  (px/run!
+   executor
+   (fn []
+     (try
+       (let [info     (retrieve-info cfg request)
+             profile  (retrieve-profile cfg info)]
+         (respond (generate-redirect cfg request info profile)))
+       (catch Exception cause
+         (l/warn :hint "error on oauth process" :cause cause)
+         (respond (generate-error-redirect cfg cause)))))))
 
 ;; --- INIT
 
@@ -245,15 +260,19 @@
 
 (defn wrap-handler
   [cfg handler]
-  (fn [request]
+  (fn [request respond raise]
     (let [provider (get-in request [:path-params :provider])
           provider (get-in @cfg [:providers provider])]
-      (when-not provider
-        (ex/raise :type :not-found
-                  :context {:provider provider}
-                  :hint "provider not configured"))
-      (-> (assoc @cfg :provider provider)
-          (handler request)))))
+      (if provider
+        (handler (assoc @cfg :provider provider)
+                 request
+                 respond
+                 raise)
+        (raise
+         (ex/error
+          :type :not-found
+          :provider provider
+          :hint "provider not configured"))))))
 
 (defmethod ig/init-key ::handler
   [_ cfg]
@@ -263,14 +282,29 @@
 
 (defn- discover-oidc-config
   [{:keys [base-uri] :as opts}]
+
   (let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
-        response      (http/send! {:method :get :uri (str discovery-uri)})]
-    (when (= 200 (:status response))
+        response      (ex/try (http/send! {:method :get :uri (str discovery-uri)}))]
+    (cond
+      (ex/exception? response)
+      (do
+        (l/warn :hint "unable to discover oidc configuration"
+                :discover-uri (str discovery-uri)
+                :cause response)
+        nil)
+
+      (= 200 (:status response))
       (let [data (json/read-str (:body response))]
-        (assoc opts
-               :token-uri (get data "token_endpoint")
-               :auth-uri (get data "authorization_endpoint")
-               :user-uri (get data "userinfo_endpoint"))))))
+        {:token-uri (get data "token_endpoint")
+         :auth-uri  (get data "authorization_endpoint")
+         :user-uri  (get data "userinfo_endpoint")})
+
+      :else
+      (do
+        (l/warn :hint "unable to discover OIDC configuration"
+                :uri (str discovery-uri)
+                :response-status-code (:status response))
+        nil))))
 
 (defn- obfuscate-string
   [s]
@@ -294,17 +328,23 @@
     (if (and (string? (:base-uri opts))
              (string? (:client-id opts))
              (string? (:client-secret opts)))
-      (if (and (string? (:token-uri opts))
-               (string? (:user-uri opts))
-               (string? (:auth-uri opts)))
-        (do
-          (l/info :action "initialize" :provider "oidc" :method "static"
-                  :opts (pr-str (update opts :client-secret obfuscate-string)))
-          (assoc-in cfg [:providers "oidc"] opts))
-        (let [opts (discover-oidc-config opts)]
-          (l/info :action "initialize" :provider "oidc" :method "discover"
-                  :opts (pr-str (update opts :client-secret obfuscate-string)))
-          (assoc-in cfg [:providers "oidc"] opts)))
+      (do
+        (l/debug :hint "initialize oidc provider" :name "generic-oidc"
+                 :opts (update opts :client-secret obfuscate-string))
+        (if (and (string? (:token-uri opts))
+                 (string? (:user-uri opts))
+                 (string? (:auth-uri opts)))
+          (do
+            (l/debug :hint "initialized with user provided configuration")
+            (assoc-in cfg [:providers "oidc"] opts))
+          (do
+            (l/debug :hint "trying to discover oidc provider configuration using BASE_URI")
+            (if-let [opts' (discover-oidc-config opts)]
+              (do
+                (l/debug :hint "discovered opts" :additional-opts opts')
+                (assoc-in cfg [:providers "oidc"] (merge opts opts')))
+
+              cfg))))
       cfg)))
 
 (defn- initialize-google-provider

backend/src/app/http/session.clj

@@ -8,92 +8,170 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.config :as cfg]
    [app.db :as db]
+   [app.db.sql :as sql]
    [app.metrics :as mtx]
    [app.util.async :as aa]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [ring.middleware.session.store :as rss]))
 
-;; A default cookie name for storing the session. We don't allow
-;; configure it.
-(def cookie-name "auth-token")
+;; A default cookie name for storing the session. We don't allow to configure it.
+(def token-cookie-name "auth-token")
+
+;; A cookie that we can use to check from other sites of the same domain if a user
+;; is registered. Is not intended for on premise installations, although nothing
+;; prevents using it if some one wants to.
+(def authenticated-cookie-name "authenticated")
+
+(deftype DatabaseStore [pool tokens]
+  rss/SessionStore
+  (read-session [_ token]
+    (db/exec-one! pool (sql/select :http-session {:id token})))
+
+  (write-session [_ _ data]
+    (let [profile-id (:profile-id data)
+          user-agent (:user-agent data)
+          token      (tokens :generate {:iss "authentication"
+                                        :iat (dt/now)
+                                        :uid profile-id})
+
+          now        (dt/now)
+          params     {:user-agent user-agent
+                      :profile-id profile-id
+                      :created-at now
+                      :updated-at now
+                      :id token}]
+      (db/insert! pool :http-session params)
+      token))
+
+  (delete-session [_ token]
+    (db/delete! pool :http-session {:id token})
+    nil))
+
+(deftype MemoryStore [cache tokens]
+  rss/SessionStore
+  (read-session [_ token]
+    (get @cache token))
+
+  (write-session [_ _ data]
+    (let [profile-id (:profile-id data)
+          user-agent (:user-agent data)
+          token      (tokens :generate {:iss "authentication"
+                                        :iat (dt/now)
+                                        :uid profile-id})
+          params     {:user-agent user-agent
+                      :profile-id profile-id
+                      :id token}]
+
+      (swap! cache assoc token params)
+      token))
+
+  (delete-session [_ token]
+    (swap! cache dissoc token)
+    nil))
 
 ;; --- IMPL
 
 (defn- create-session
-  [{:keys [conn tokens] :as cfg} {:keys [profile-id headers] :as request}]
-  (let [token  (tokens :generate {:iss "authentication"
-                                  :iat (dt/now)
-                                  :uid profile-id})
-        params {:user-agent (get headers "user-agent")
-                :profile-id profile-id
-                :id token}]
-    (db/insert! conn :http-session params)))
+  [store request profile-id]
+  (let [params {:user-agent (get-in request [:headers "user-agent"])
+                :profile-id profile-id}]
+    (rss/write-session store nil params)))
 
 (defn- delete-session
-  [{:keys [conn] :as cfg} {:keys [cookies] :as request}]
-  (when-let [token (get-in cookies [cookie-name :value])]
-    (db/delete! conn :http-session {:id token}))
-  nil)
+  [store {:keys [cookies] :as request}]
+  (when-let [token (get-in cookies [token-cookie-name :value])]
+    (rss/delete-session store token)))
 
 (defn- retrieve-session
-  [{:keys [conn] :as cfg} id]
-  (when id
-    (db/exec-one! conn ["select id, profile_id from http_session where id = ?" id])))
+  [store token]
+  (when token
+    (rss/read-session store token)))
 
 (defn- retrieve-from-request
-  [cfg {:keys [cookies] :as request}]
-  (->> (get-in cookies [cookie-name :value])
-       (retrieve-session cfg)))
+  [store {:keys [cookies] :as request}]
+  (->> (get-in cookies [token-cookie-name :value])
+       (retrieve-session store)))
 
 (defn- add-cookies
-  [response {:keys [id] :as session}]
-  (assoc response :cookies {cookie-name {:path "/" :http-only true :value id}}))
+  [response token]
+  (let [cors?   (contains? cfg/flags :cors)
+        secure? (contains? cfg/flags :secure-session-cookies)
+        authenticated-cookie-domain (cfg/get :authenticated-cookie-domain)]
+    (update response :cookies
+            (fn [cookies]
+              (cond-> cookies
+                :always
+                (assoc token-cookie-name {:path "/"
+                                          :http-only true
+                                          :value token
+                                          :same-site (if cors? :none :lax)
+                                          :secure secure?})
+
+                (some? authenticated-cookie-domain)
+                (assoc authenticated-cookie-name {:domain authenticated-cookie-domain
+                                                  :path "/"
+                                                  :value true
+                                                  :same-site :strict
+                                                  :secure secure?}))))))
 
 (defn- clear-cookies
   [response]
-  (assoc response :cookies {cookie-name {:value "" :max-age -1}}))
+  (let [authenticated-cookie-domain (cfg/get :authenticated-cookie-domain)]
+    (assoc response :cookies {token-cookie-name {:path "/"
+                                                 :value ""
+                                                 :max-age -1}
+                              authenticated-cookie-name {:domain authenticated-cookie-domain
+                                                         :path "/"
+                                                         :value ""
+                                                         :max-age -1}})))
 
 (defn- middleware
-  [cfg handler]
-  (fn [request]
-    (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
+  [events-ch store handler]
+  (fn [request respond raise]
+    (if-let [{:keys [id profile-id] :as session} (retrieve-from-request store request)]
       (do
-        (a/>!! (::events-ch cfg) id)
-        (l/update-thread-context! {:profile-id profile-id})
-        (handler (assoc request :profile-id profile-id)))
-      (handler request))))
+        (a/>!! events-ch id)
+        (l/set-context! {:profile-id profile-id})
+        (handler (assoc request :profile-id profile-id :session-id id) respond raise))
+      (handler request respond raise))))
 
 ;; --- STATE INIT: SESSION
 
+(s/def ::tokens fn?)
 (defmethod ig/pre-init-spec ::session [_]
-  (s/keys :req-un [::db/pool]))
+  (s/keys :req-un [::db/pool ::tokens]))
 
 (defmethod ig/prep-key ::session
   [_ cfg]
-  (d/merge {:buffer-size 64}
+  (d/merge {:buffer-size 128}
            (d/without-nils cfg)))
 
 (defmethod ig/init-key ::session
-  [_ {:keys [pool] :as cfg}]
-  (let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
-        cfg    (-> cfg
-                   (assoc :conn pool)
-                   (assoc ::events-ch events))]
+  [_ {:keys [pool tokens] :as cfg}]
+  (let [events-ch (a/chan (a/dropping-buffer (:buffer-size cfg)))
+        store     (if (db/read-only? pool)
+                    (->MemoryStore (atom {}) tokens)
+                    (->DatabaseStore pool tokens))]
+
+    (when (db/read-only? pool)
+      (l/warn :hint "sessions module initialized with in-memory store"))
+
     (-> cfg
-        (assoc :middleware #(middleware cfg %))
+        (assoc ::events-ch events-ch)
+        (assoc :middleware (partial middleware events-ch store))
         (assoc :create (fn [profile-id]
                          (fn [request response]
-                           (let [request (assoc request :profile-id profile-id)
-                                 session (create-session cfg request)]
-                             (add-cookies response session)))))
+                           (let [token (create-session store request profile-id)]
+                             (add-cookies response token)))))
         (assoc :delete (fn [request response]
-                         (delete-session cfg request)
+                         (delete-session store request)
                          (-> response
                              (assoc :status 204)
                              (assoc :body "")
@@ -130,16 +208,11 @@
           :max-batch-size (str (:max-batch-size cfg)))
   (let [input (aa/batch (::events-ch session)
                         {:max-batch-size (:max-batch-size cfg)
-                         :max-batch-age (inst-ms (:max-batch-age cfg))})
-        mcnt  (mtx/create
-               {:name "http_session_update_total"
-                :help "A counter of session update batch events."
-                :registry (:registry metrics)
-                :type :counter})]
+                         :max-batch-age (inst-ms (:max-batch-age cfg))})]
     (a/go-loop []
       (when-let [[reason batch] (a/<! input)]
         (let [result (a/<! (update-sessions cfg batch))]
-          (mcnt :inc)
+          (mtx/run! metrics {:id :session-update-total :inc 1})
           (cond
             (ex/exception? result)
             (l/error :task "updater"
@@ -148,9 +221,10 @@
 
             (= :size reason)
             (l/debug :task "updater"
-                     :action "update sessions"
+                     :hint "update sessions"
                      :reason (name reason)
                      :count result))
+
           (recur))))))
 
 (defn- update-sessions
@@ -172,22 +246,25 @@
 
 (defmethod ig/prep-key ::gc-task
   [_ cfg]
-  (merge {:max-age (dt/duration {:days 2})}
+  (merge {:max-age (dt/duration {:days 15})}
          (d/without-nils cfg)))
 
 (defmethod ig/init-key ::gc-task
   [_ {:keys [pool max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
   (fn [_]
     (db/with-atomic [conn pool]
       (let [interval (db/interval max-age)
-            result   (db/exec-one! conn [sql:delete-expired interval])
+            result   (db/exec-one! conn [sql:delete-expired interval interval])
             result   (:next.jdbc/update-count result)]
         (l/debug :task "gc"
-                 :action "clean http sessions"
-                 :count result)
+                 :hint "clean http sessions"
+                 :deleted result)
         result))))
 
 (def ^:private
   sql:delete-expired
   "delete from http_session
-    where updated_at < now() - ?::interval")
+    where updated_at < now() - ?::interval
+       or (updated_at is null and
+           created_at < now() - ?::interval)")

backend/src/app/http/websocket.clj

@@ -0,0 +1,144 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.websocket
+  "A penpot notification service for file cooperative edition."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.metrics :as mtx]
+   [app.util.websocket :as ws]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [yetti.websocket :as yws]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; WEBSOCKET HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare send-presence!)
+
+(defmulti handle-message
+  (fn [_wsp message] (:type message)))
+
+(defmethod handle-message :connect
+  [wsp _]
+  (let [{:keys [msgbus file-id team-id session-id ::ws/output-ch]} @wsp
+        sub-ch (a/chan (a/dropping-buffer 32))]
+
+    (swap! wsp assoc :sub-ch sub-ch)
+
+    ;; Start a subscription forwarding goroutine
+    (a/go-loop []
+      (when-let [val (a/<! sub-ch)]
+        (when-not (= (:session-id val) session-id)
+          ;; If we receive a connect message of other user, we need
+          ;; to send an update presence to all participants.
+          (when (= :connect (:type val))
+            (a/<! (send-presence! @wsp :presence)))
+
+          ;; Then, just forward the message
+          (a/>! output-ch val))
+        (recur)))
+
+    (a/go
+      (a/<! (msgbus :sub {:topics [file-id team-id] :chan sub-ch}))
+      (a/<! (send-presence! @wsp :connect)))))
+
+(defmethod handle-message :disconnect
+  [wsp _]
+  (a/close! (:sub-ch @wsp))
+  (send-presence! @wsp :disconnect))
+
+(defmethod handle-message :keepalive
+  [_ _]
+  (a/go :nothing))
+
+(defmethod handle-message :pointer-update
+  [wsp message]
+  (let [{:keys [profile-id file-id session-id msgbus]} @wsp]
+    (msgbus :pub {:topic file-id
+                  :message (assoc message
+                                  :profile-id profile-id
+                                  :session-id session-id)})))
+
+(defmethod handle-message :default
+  [_ message]
+  (a/go
+    (l/log :level :warn
+           :msg "received unexpected message"
+           :message message)))
+
+;; --- IMPL
+
+(defn- send-presence!
+  ([ws] (send-presence! ws :presence))
+  ([{:keys [msgbus session-id profile-id file-id]} type]
+   (msgbus :pub {:topic file-id
+                 :message {:type type
+                           :session-id session-id
+                           :profile-id profile-id}})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare retrieve-file)
+
+(s/def ::msgbus fn?)
+(s/def ::file-id ::us/uuid)
+(s/def ::session-id ::us/uuid)
+
+(s/def ::handler-params
+  (s/keys :req-un [::file-id ::session-id]))
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::msgbus ::db/pool ::mtx/metrics]))
+
+(defmethod ig/init-key ::handler
+  [_ {:keys [pool] :as cfg}]
+  (fn [{:keys [profile-id params] :as req} respond raise]
+    (let [params (us/conform ::handler-params params)
+          file   (retrieve-file pool (:file-id params))
+          cfg    (-> (merge cfg params)
+                     (assoc :profile-id profile-id)
+                     (assoc :team-id (:team-id file)))]
+
+      (cond
+        (not profile-id)
+        (raise (ex/error :type :authentication
+                         :hint "Authentication required."))
+
+        (not file)
+        (raise (ex/error :type :not-found
+                         :code :object-not-found))
+
+
+        (not (yws/upgrade-request? req))
+        (raise (ex/error :type :validation
+                         :code :websocket-request-expected
+                         :hint "this endpoint only accepts websocket connections"))
+
+        :else
+        (->> (ws/handler handle-message cfg)
+             (yws/upgrade req)
+             (respond))))))
+
+(def ^:private
+  sql:retrieve-file
+  "select f.id as id,
+          p.team_id as team_id
+     from file as f
+     join project as p on (p.id = f.project_id)
+    where f.id = ?")
+
+(defn- retrieve-file
+  [conn id]
+  (db/exec-one! conn [sql:retrieve-file id]))
+

backend/src/app/loggers/audit.clj

@@ -9,6 +9,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
@@ -16,14 +17,15 @@
    [app.db :as db]
    [app.util.async :as aa]
    [app.util.http :as http]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [lambdaisland.uri :as u]))
+   [lambdaisland.uri :as u]
+   [promesa.core :as p]
+   [promesa.exec :as px]))
 
 (defn parse-client-ip
   [{:keys [headers] :as request}]
@@ -35,37 +37,101 @@
   [profile]
   (-> profile
       (select-keys [:is-active :is-muted :auth-backend :email :default-team-id :default-project-id :fullname :lang])
+      (merge (:props profile))
       (d/without-nils)))
 
 (defn clean-props
   [{:keys [profile-id] :as event}]
-  (letfn [(clean-common [props]
-            (-> props
-                (dissoc :session-id)
-                (dissoc :password)
-                (dissoc :old-password)
-                (dissoc :token)))
+  (let [invalid-keys #{:session-id
+                       :password
+                       :old-password
+                       :token}
+        xform (comp
+               (remove (fn [kv]
+                         (qualified-keyword? (first kv))))
+               (remove (fn [kv]
+                         (contains? invalid-keys (first kv))))
+               (remove (fn [[k v]]
+                         (and (= k :profile-id)
+                              (= v profile-id))))
+               (filter (fn [[_ v]]
+                         (or (string? v)
+                             (keyword? v)
+                             (uuid? v)
+                             (boolean? v)
+                             (number? v)))))]
 
-          (clean-profile-id [props]
-            (cond-> props
-              (= profile-id (:profile-id props))
-              (dissoc :profile-id)))
+    (update event :props #(into {} xform %))))
 
-          (clean-complex-data [props]
-            (reduce-kv (fn [props k v]
-                         (cond-> props
-                           (or (string? v)
-                               (uuid? v)
-                               (boolean? v)
-                               (number? v))
-                           (assoc k v)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP Handler
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-                           (keyword? v)
-                           (assoc k (name v))))
-                       {}
-                       props))]
+(declare persist-http-events)
 
-    (update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
+(s/def ::profile-id ::us/uuid)
+(s/def ::name ::us/string)
+(s/def ::type ::us/string)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::timestamp dt/instant?)
+(s/def ::context (s/map-of ::us/keyword any?))
+
+(s/def ::frontend-event
+  (s/keys :req-un [::type ::name ::props ::timestamp ::profile-id]
+          :opt-un [::context]))
+
+(s/def ::frontend-events (s/every ::frontend-event))
+
+(defmethod ig/init-key ::http-handler
+  [_  {:keys [executor pool] :as cfg}]
+  (if (or (db/read-only? pool) (not (contains? cf/flags :audit-log)))
+    (do
+      (l/warn :hint "audit log http handler disabled or db is read-only")
+      (fn [_ respond _]
+        (respond {:status 204 :body ""})))
+
+
+    (letfn [(handler [{:keys [params profile-id] :as request}]
+              (let [events  (->> (:events params)
+                                 (remove #(not= profile-id (:profile-id %)))
+                                 (us/conform ::frontend-events))
+
+                    ip-addr (parse-client-ip request)
+                    cfg     (-> cfg
+                                (assoc :source "frontend")
+                                (assoc :events events)
+                                (assoc :ip-addr ip-addr))]
+                (persist-http-events cfg)))
+
+            (handle-error [cause]
+              (let [xdata (ex-data cause)]
+                (if (= :spec-validation (:code xdata))
+                  (l/error ::l/raw (str "spec validation on persist-events:\n" (us/pretty-explain xdata)))
+                  (l/error :hint "error on persist-events" :cause cause))))]
+
+      (fn [request respond _]
+        ;; Fire and forget, log error in case of errro
+        (-> (px/submit! executor #(handler request))
+            (p/catch handle-error))
+
+        (respond {:status 204 :body ""})))))
+
+(defn- persist-http-events
+  [{:keys [pool events ip-addr source] :as cfg}]
+  (let [columns    [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context]
+        prepare-xf (map (fn [event]
+                          [(uuid/next)
+                           (:name event)
+                           source
+                           (:type event)
+                           (:timestamp event)
+                           (:profile-id event)
+                           (db/inet ip-addr)
+                           (db/tjson (:props event))
+                           (db/tjson (d/without-nils (:context event)))]))]
+    (when (seq events)
+      (->> (into [] prepare-xf events)
+           (db/insert-multi! pool :audit-log columns)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Collector
@@ -76,39 +142,57 @@
 ;; an external storage and data cleared.
 
 (declare persist-events)
-(s/def ::enabled ::us/boolean)
 
 (defmethod ig/pre-init-spec ::collector [_]
-  (s/keys :req-un [::db/pool ::wrk/executor ::enabled]))
+  (s/keys :req-un [::db/pool ::wrk/executor]))
 
-(def event-xform
+(s/def ::ip-addr string?)
+(s/def ::backend-event
+  (s/keys :req-un [::type ::name ::profile-id]
+          :opt-un [::ip-addr ::props]))
+
+(def ^:private backend-event-xform
   (comp
-   (filter :profile-id)
+   (filter #(us/valid? ::backend-event %))
    (map clean-props)))
 
 (defmethod ig/init-key ::collector
-  [_ {:keys [enabled] :as cfg}]
-  (when enabled
-    (l/info :msg "intializing audit collector")
-    (let [input  (a/chan 512 event-xform)
+  [_ {:keys [pool] :as cfg}]
+  (cond
+    (not (contains? cf/flags :audit-log))
+    (do
+      (l/info :hint "audit log collection disabled")
+      (constantly nil))
+
+    (db/read-only? pool)
+    (do
+      (l/warn :hint "audit log collection disabled, db is read-only")
+      (constantly nil))
+
+    :else
+    (let [input  (a/chan 512 backend-event-xform)
           buffer (aa/batch input {:max-batch-size 100
                                   :max-batch-age (* 10 1000) ; 10s
                                   :init []})]
+      (l/info :hint "audit log collector initialized")
       (a/go-loop []
         (when-let [[_type events] (a/<! buffer)]
           (let [res (a/<! (persist-events cfg events))]
             (when (ex/exception? res)
-              (l/error :hint "error on persiting events"
-                       :cause res)))
-          (recur)))
+              (l/error :hint "error on persisting events" :cause res))
+            (recur))))
 
       (fn [& {:keys [cmd] :as params}]
-        (let [params (dissoc params :cmd)]
-          (case cmd
-            :stop   (a/close! input)
-            :submit (when-not (a/offer! input params)
-                      (l/warn :msg "activity channel is full"))))))))
+        (case cmd
+          :stop
+          (a/close! input)
 
+          :submit
+          (let [params (-> params
+                           (dissoc :cmd)
+                           (assoc :tracked-at (dt/now)))]
+            (when-not (a/offer! input params)
+              (l/warn :hint "activity channel is full"))))))))
 
 (defn- persist-events
   [{:keys [pool executor] :as cfg} events]
@@ -117,20 +201,22 @@
              (:name event)
              (:type event)
              (:profile-id event)
+             (:tracked-at event)
              (some-> (:ip-addr event) db/inet)
-             (db/tjson (:props event))])]
-
+             (db/tjson (:props event))
+             "backend"])]
     (aa/with-thread executor
-      (db/with-atomic [conn pool]
-        (db/insert-multi! conn :audit-log
-                          [:id :name :type :profile-id :ip-addr :props]
-                          (sequence (map event->row) events))))))
+      (when (seq events)
+        (db/with-atomic [conn pool]
+          (db/insert-multi! conn :audit-log
+                            [:id :name :type :profile-id :tracked-at :ip-addr :props :source]
+                            (sequence (keep event->row) events)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Archive Task
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; This is a task responsible to send the accomulated events to an
+;; This is a task responsible to send the accumulated events to an
 ;; external service for archival.
 
 (declare archive-events)
@@ -139,47 +225,60 @@
 (s/def ::tokens fn?)
 
 (defmethod ig/pre-init-spec ::archive-task [_]
-  (s/keys :req-un [::db/pool ::tokens ::enabled]
+  (s/keys :req-un [::db/pool ::tokens]
           :opt-un [::uri]))
 
 (defmethod ig/init-key ::archive-task
-  [_ {:keys [uri enabled] :as cfg}]
-  (fn [_]
-    (when (and enabled (not uri))
-      (ex/raise :type :internal
-                :code :task-not-configured
-                :hint "archive task not configured, missing uri"))
-    (loop []
-      (let [res (archive-events cfg)]
-        (when (= res :continue)
-          (aa/thread-sleep 200)
-          (recur))))))
+  [_ {:keys [uri] :as cfg}]
+  (fn [props]
+    ;; NOTE: this let allows overwrite default configured values from
+    ;; the repl, when manually invoking the task.
+    (let [enabled (or (contains? cf/flags :audit-log-archive)
+                      (:enabled props false))
+          uri     (or uri (:uri props))
+          cfg     (assoc cfg :uri uri)]
+
+      (when (and enabled (not uri))
+        (ex/raise :type :internal
+                  :code :task-not-configured
+                  :hint "archive task not configured, missing uri"))
+      (when enabled
+        (loop []
+          (let [res (archive-events cfg)]
+            (when (= res :continue)
+              (aa/thread-sleep 200)
+              (recur))))))))
 
 (def sql:retrieve-batch-of-audit-log
   "select * from audit_log
     where archived_at is null
     order by created_at asc
-    limit 100
+    limit 1000
       for update skip locked;")
 
 (defn archive-events
   [{:keys [pool uri tokens] :as cfg}]
-  (letfn [(decode-row [{:keys [props ip-addr] :as row}]
+  (letfn [(decode-row [{:keys [props ip-addr context] :as row}]
             (cond-> row
               (db/pgobject? props)
               (assoc :props (db/decode-transit-pgobject props))
 
+              (db/pgobject? context)
+              (assoc :context (db/decode-transit-pgobject context))
+
               (db/pgobject? ip-addr "inet")
               (assoc :ip-addr (db/decode-inet ip-addr))))
 
-          (row->event [{:keys [name type created-at profile-id props ip-addr]}]
-            (cond-> {:type type
-                     :name name
-                     :timestamp created-at
-                     :profile-id profile-id
-                     :props props}
-              (some? ip-addr)
-              (update :context assoc :source-ip ip-addr)))
+          (row->event [row]
+            (select-keys row [:type
+                              :name
+                              :source
+                              :created-at
+                              :tracked-at
+                              :profile-id
+                              :ip-addr
+                              :props
+                              :context]))
 
           (send [events]
             (let [token   (tokens :generate {:iss "authentication"
@@ -195,11 +294,12 @@
                            :headers headers
                            :body body}
                   resp    (http/send! params)]
-              (when (not= (:status resp) 204)
-                (ex/raise :type :internal
-                          :code :unable-to-send-events
-                          :hint "unable to send events"
-                          :context resp))))
+              (if (= (:status resp) 204)
+                true
+                (do
+                  (l/warn :hint "unable to archive events"
+                          :resp-status (:status resp))
+                  false))))
 
           (mark-as-archived [conn rows]
             (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)"
@@ -212,11 +312,9 @@
             xform  (comp (map decode-row)
                          (map row->event))
             events (into [] xform rows)]
-        (l/debug :action "archive-events" :uri uri :events (count events))
-        (if (empty? events)
-          :empty
-          (do
-            (send events)
+        (when-not (empty? events)
+          (l/debug :action "archive-events" :uri uri :events (count events))
+          (when (send events)
             (mark-as-archived conn rows)
             :continue))))))
 
@@ -224,18 +322,6 @@
 ;; GC Task
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare clean-archived)
-
-(s/def ::max-age ::cf/audit-archive-gc-max-age)
-
-(defmethod ig/pre-init-spec ::archive-gc-task [_]
-  (s/keys :req-un [::db/pool ::enabled ::max-age]))
-
-(defmethod ig/init-key ::archive-gc-task
-  [_ cfg]
-  (fn [_]
-    (clean-archived cfg)))
-
 (def sql:clean-archived
   "delete from audit_log
     where archived_at is not null
@@ -248,3 +334,13 @@
         result   (:next.jdbc/update-count result)]
     (l/debug :action "clean archived audit log" :removed result)
     result))
+
+(s/def ::max-age ::cf/audit-log-gc-max-age)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool ::max-age]))
+
+(defmethod ig/init-key ::gc-task
+  [_ cfg]
+  (fn [_]
+    (clean-archived cfg)))

backend/src/app/loggers/database.clj

@@ -0,0 +1,92 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.database
+  "A specific logger impl that persists errors on the database."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare handle-event)
+
+(defonce enabled (atom true))
+
+(defn- persist-on-database!
+  [{:keys [pool] :as cfg} {:keys [id] :as event}]
+  (when-not (db/read-only? pool)
+    (db/insert! pool :server-error-report {:id id :content (db/tjson event)})))
+
+(defn- parse-event-data
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   event))
+
+(defn parse-event
+  [event]
+  (-> (parse-event-data event)
+      (assoc :tenant (cf/get :tenant))
+      (assoc :host (cf/get :host))
+      (assoc :public-uri (cf/get :public-uri))
+      (assoc :version (:full cf/version))
+      (update :id #(or % (uuid/next)))))
+
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [event (parse-event event)
+            uri   (cf/get :public-uri)]
+
+        (l/debug :hint "registering error on database" :id (:id event)
+                 :uri (str uri "/dbg/error/" (:id event)))
+
+        (persist-on-database! cfg event))
+      (catch Exception cause
+        (l/warn :hint "unexpected exception on database error logger" :cause cause)))))
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
+
+(defn error-event?
+  [event]
+  (= "error" (:logger/level event)))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver] :as cfg}]
+  (l/info :msg "initializing database error persistence")
+  (let [output (a/chan (a/sliding-buffer 5) (filter error-event?))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [msg (a/<! output)]
+        (if (nil? msg)
+          (l/info :msg "stoping error reporting loop")
+          (do
+            (a/<! (handle-event cfg msg))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (a/close! output))

backend/src/app/loggers/loki.clj

@@ -7,12 +7,12 @@
 (ns app.loggers.loki
   "A Loki integration."
   (:require
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cfg]
    [app.util.async :as aa]
    [app.util.http :as http]
    [app.util.json :as json]
-   [app.util.logging :as l]
    [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
@@ -30,7 +30,7 @@
 (defmethod ig/init-key ::reporter
   [_ {:keys [receiver uri] :as cfg}]
   (when uri
-    (l/info :msg "intializing loki reporter" :uri uri)
+    (l/info :msg "initializing loki reporter" :uri uri)
     (let [input (a/chan (a/dropping-buffer 512))]
       (receiver :sub input)
       (a/go-loop []
@@ -52,14 +52,14 @@
   (let [labels {:host    (cfg/get :host)
                 :tenant  (cfg/get :tenant)
                 :version (:full cfg/version)
-                :logger  (:logger event)
-                :level   (:level event)}]
+                :logger  (:logger/name event)
+                :level   (:logger/level event)}]
     {:streams
      [{:stream labels
        :values [[(str (* (inst-ms (:created-at event)) 1000000))
                  (str (:message event)
-                      (when-let [error (:error event)]
-                        (str "\n" (:trace error))))]]}]}))
+                      (when-let [error (:trace event)]
+                        (str "\n" error)))]]}]}))
 
 (defn- send-log
   [uri payload i]
@@ -68,7 +68,7 @@
                                 :timeout 6000
                                 :method :post
                                 :headers {"content-type" "application/json"}
-                                :body (json/encode payload)})]
+                                :body (json/write payload)})]
       (cond
         (= (:status response) 204)
         true

backend/src/app/loggers/mattermost.clj

@@ -7,32 +7,51 @@
 (ns app.loggers.mattermost
   "A mattermost integration for error reporting."
   (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
+   [app.common.logging :as l]
+   [app.config :as cf]
    [app.db :as db]
+   [app.loggers.database :as ldb]
    [app.util.async :as aa]
    [app.util.http :as http]
    [app.util.json :as json]
-   [app.util.logging :as l]
-   [app.util.template :as tmpl]
    [app.worker :as wrk]
    [clojure.core.async :as a]
-   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
    [integrant.core :as ig]))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Error Listener
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defonce enabled (atom true))
 
-(declare handle-event)
+(defn- send-mattermost-notification!
+  [cfg {:keys [host id public-uri] :as event}]
+  (try
+    (let [uri  (:uri cfg)
+          text (str "Exception on (host: " host ", url: " public-uri "/dbg/error/" id ")\n"
+                    (when-let [pid (:profile-id event)]
+                      (str "- profile-id: #uuid-" pid "\n")))
+          rsp  (http/send! {:uri uri
+                            :method :post
+                            :headers {"content-type" "application/json"}
+                            :body (json/write-str {:text text})})]
+      (when (not= (:status rsp) 200)
+        (l/error :hint "error on sending data to mattermost"
+                 :response (pr-str rsp))))
 
-(defonce enabled-mattermost (atom true))
+    (catch Exception e
+      (l/error :hint "unexpected exception on error reporter"
+               :cause e))))
 
-(s/def ::uri ::us/string)
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [event (ldb/parse-event event)]
+        (when @enabled
+          (send-mattermost-notification! cfg event)))
+      (catch Exception e
+        (l/warn :hint "unexpected exception on error reporter" :cause e)))))
+
+
+(s/def ::uri ::cf/error-report-webhook)
 
 (defmethod ig/pre-init-spec ::reporter [_]
   (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
@@ -43,7 +62,8 @@
   (when uri
     (l/info :msg "initializing mattermost error reporter" :uri uri)
     (let [output (a/chan (a/sliding-buffer 128)
-                         (filter #(= (:level %) "error")))]
+                         (filter (fn [event]
+                                   (= (:logger/level event) "error"))))]
       (receiver :sub output)
       (a/go-loop []
         (let [msg (a/<! output)]
@@ -58,95 +78,3 @@
   [_ output]
   (when output
     (a/close! output)))
-
-(defn- send-mattermost-notification!
-  [cfg {:keys [host id] :as cdata}]
-  (try
-    (let [uri  (:uri cfg)
-          text (str "Unhandled exception (host: " host ", url: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
-                    "- profile-id: #" (:profile-id cdata) "\n")
-          rsp  (http/send! {:uri uri
-                            :method :post
-                            :headers {"content-type" "application/json"}
-                            :body (json/encode-str {:text text})})]
-      (when (not= (:status rsp) 200)
-        (l/error :hint "error on sending data to mattermost"
-                 :response (pr-str rsp))))
-
-    (catch Exception e
-      (l/error :hint "unexpected exception on error reporter"
-               :cause e))))
-
-(defn- persist-on-database!
-  [{:keys [pool] :as cfg} {:keys [id] :as cdata}]
-  (db/with-atomic [conn pool]
-    (db/insert! conn :server-error-report
-                {:id id :content (db/tjson cdata)})))
-
-(defn- parse-context
-  [event]
-  (reduce-kv
-   (fn [acc k v]
-     (cond
-       (= k :id)         (assoc acc k (uuid/uuid v))
-       (= k :profile-id) (assoc acc k (uuid/uuid v))
-       (str/blank? v)    acc
-       :else             (assoc acc k v)))
-   {:id (uuid/next)}
-   (:context event)))
-
-(defn- parse-event
-  [event]
-  (-> (parse-context event)
-      (merge (dissoc event :context))
-      (assoc :tenant (cfg/get :tenant))
-      (assoc :host (cfg/get :host))
-      (assoc :public-uri (cfg/get :public-uri))
-      (assoc :version (:full cfg/version))))
-
-(defn handle-event
-  [{:keys [executor] :as cfg} event]
-  (aa/with-thread executor
-    (try
-      (let [cdata (parse-event event)]
-        (when @enabled-mattermost
-          (send-mattermost-notification! cfg cdata))
-        (persist-on-database! cfg cdata))
-      (catch Exception e
-        (l/error :hint "unexpected exception on error reporter"
-                 :cause e)))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Handler
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as cfg}]
-  (letfn [(parse-id [request]
-            (let [id (get-in request [:path-params :id])
-                  id (us/uuid-conformer id)]
-              (when (uuid? id)
-                id)))
-          (retrieve-report [id]
-            (ex/ignoring
-             (when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
-               (assoc row :content (db/decode-transit-pgobject content)))))
-
-          (render-template [{:keys [content] :as report}]
-            (some-> (io/resource "error-report.tmpl")
-                    (tmpl/render content)))]
-
-
-    (fn [request]
-      (let [result (some-> (parse-id request)
-                           (retrieve-report)
-                           (render-template))]
-        (if result
-          {:status 200
-           :headers {"content-type" "text/html; charset=utf-8"}
-           :body result}
-          {:status 404
-           :body "not found"})))))

backend/src/app/loggers/sentry.clj

@@ -0,0 +1,170 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.sentry
+  "A mattermost integration for error reporting."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig])
+  (:import
+   io.sentry.Scope
+   io.sentry.IHub
+   io.sentry.Hub
+   io.sentry.NoOpHub
+   io.sentry.protocol.User
+   io.sentry.SentryOptions
+   io.sentry.SentryLevel
+   io.sentry.ScopeCallback))
+
+(defonce enabled (atom true))
+
+(defn- parse-context
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   (:context event)))
+
+(defn- parse-event
+  [event]
+  (assoc event :context (parse-context event)))
+
+(defn- build-sentry-options
+  [cfg]
+  (let [version (:base cf/version)]
+    (doto (SentryOptions.)
+      (.setDebug (:debug cfg false))
+      (.setTracesSampleRate (:traces-sample-rate cfg 1.0))
+      (.setDsn (:dsn cfg))
+      (.setServerName (cf/get :host))
+      (.setEnvironment (cf/get :tenant))
+      (.setAttachServerName true)
+      (.setAttachStacktrace (:attach-stack-trace cfg false))
+      (.setRelease (str "backend@" (if (= version "0.0.0") "develop" version))))))
+
+(defn handle-event
+  [^IHub shub event]
+  (letfn [(set-user! [^Scope scope {:keys [context] :as event}]
+            (let [user (User.)]
+              (.setIpAddress ^User user ^String (:ip-addr context))
+              (when-let [pid (:profile-id context)]
+                (.setId ^User user ^String (str pid)))
+              (.setUser scope ^User user)))
+
+          (set-level! [^Scope scope]
+            (.setLevel scope SentryLevel/ERROR))
+
+          (set-context! [^Scope scope {:keys [context] :as event}]
+            (let [uri (str (cf/get :public-uri) "/dbg/error-by-id/" (:id context))]
+              (.setContexts scope "detailed_error_uri" ^String uri))
+            (when-let [vers (:frontend-version event)]
+              (.setContexts scope "frontend_version" ^String vers))
+            (when-let [puri (:public-uri event)]
+              (.setContexts scope "public_uri" ^String (str puri)))
+            (when-let [uagent (:user-agent context)]
+              (.setContexts scope "user_agent" ^String uagent))
+            (when-let [tenant (:tenant event)]
+              (.setTag scope "tenant" ^String tenant))
+            (when-let [type (:error-type context)]
+              (.setTag scope "error_type" ^String  (str type)))
+            (when-let [code (:error-code context)]
+              (.setTag scope "error_code" ^String (str code)))
+            )
+
+          (capture [^Scope scope {:keys [context error] :as event}]
+            (let [msg   (str (:message error) "\n\n"
+
+                             "======================================================\n"
+                             "=================== Params ===========================\n"
+                             "======================================================\n"
+
+                             (:params context) "\n"
+
+                             (when (:explain context)
+                               (str "======================================================\n"
+                                    "=================== Explain ==========================\n"
+                                    "======================================================\n"
+                                    (:explain context) "\n"))
+
+                             (when (:data context)
+                               (str "======================================================\n"
+                                    "=================== Error Data =======================\n"
+                                    "======================================================\n"
+                                    (:data context) "\n"))
+
+                             (str "======================================================\n"
+                                  "=================== Stack Trace ======================\n"
+                                  "======================================================\n"
+                                  (:trace error))
+
+                             "\n")]
+              (set-user! scope event)
+              (set-level! scope)
+              (set-context! scope event)
+              (.captureMessage ^IHub shub msg)
+              ))
+          ]
+    (when @enabled
+      (.withScope ^IHub shub (reify ScopeCallback
+                               (run [_ scope]
+                                 (->> event
+                                      (parse-event)
+                                      (capture scope))))))
+
+    ))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::receiver any?)
+(s/def ::dsn ::cf/sentry-dsn)
+(s/def ::trace-sample-rate ::cf/sentry-trace-sample-rate)
+(s/def ::attach-stack-trace ::cf/sentry-attach-stack-trace)
+(s/def ::debug ::cf/sentry-debug)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+          :opt-un [::dsn ::trace-sample-rate ::attach-stack-trace]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver dsn executor] :as cfg}]
+  (l/info :msg "initializing sentry reporter" :dsn dsn)
+  (let [opts   (build-sentry-options cfg)
+        shub   (if dsn
+                 (Hub. ^SentryOptions opts)
+                 (NoOpHub/getInstance))
+        output (a/chan (a/sliding-buffer 128)
+                       (filter #(= (:level %) "error")))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [event (a/<! output)]
+        (if (nil? event)
+          (do
+            (l/info :msg "stoping error reporting loop")
+            (.close ^IHub shub))
+          (do
+            (a/<! (aa/with-thread executor (handle-event shub event)))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))

backend/src/app/loggers/zmq.clj

@@ -7,10 +7,10 @@
 (ns app.loggers.zmq
   "A generic ZMQ listener."
   (:require
-   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.util.json :as json]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
@@ -31,10 +31,10 @@
 
 (defmethod ig/init-key ::receiver
   [_ {:keys [endpoint] :as cfg}]
-  (l/info :msg "intializing ZMQ receiver" :bind endpoint)
+  (l/info :msg "initializing ZMQ receiver" :bind endpoint)
   (let [buffer (a/chan 1)
         output (a/chan 1 (comp (filter map?)
-                               (map prepare)))
+                               (keep prepare)))
         mult   (a/mult output)]
     (when endpoint
       (a/thread (start-rcv-loop {:out buffer :endpoint endpoint})))
@@ -53,6 +53,11 @@
   [_ f]
   (a/close! (::buffer (meta f))))
 
+(def ^:private json-mapper
+  (json/mapper
+   {:encode-key-fn str/camel
+    :decode-key-fn (comp keyword str/kebab)}))
+
 (defn- start-rcv-loop
   ([] (start-rcv-loop nil))
   ([{:keys [out endpoint] :or {endpoint "tcp://localhost:5556"}}]
@@ -64,7 +69,7 @@
      (.. socket (setReceiveTimeOut 5000))
      (loop []
        (let [msg (.recv ^ZMQ$Socket socket)
-             msg (json/decode msg)
+             msg (ex/ignoring (json/read msg json-mapper))
              msg (if (nil? msg) :empty msg)]
          (if (a/>!! out msg)
            (recur)
@@ -72,18 +77,30 @@
              (.close ^java.lang.AutoCloseable socket)
              (.close ^java.lang.AutoCloseable zctx))))))))
 
+(s/def ::logger-name string?)
+(s/def ::level string?)
+(s/def ::thread string?)
+(s/def ::time-millis integer?)
+(s/def ::message string?)
+(s/def ::context-map map?)
+(s/def ::throw map?)
+
+(s/def ::log4j-event
+  (s/keys :req-un [::logger-name ::level ::thread ::time-millis ::message]
+          :opt-un [::context-map ::thrown]))
+
 (defn- prepare
   [event]
-  (d/merge
-   {:logger     (:loggerName event)
-    :level      (str/lower (:level event))
-    :thread     (:thread event)
-    :created-at (dt/instant (:timeMillis event))
-    :message    (:message event)}
-   (when-let [ctx (:contextMap event)]
-     {:context ctx})
-   (when-let [thrown (:thrown event)]
-     {:error
-      {:class (:name thrown)
-       :message (:message thrown)
-       :trace (:extendedStackTrace thrown)}})))
+  (if (s/valid? ::log4j-event event)
+    (merge {:message      (:message event)
+            :created-at   (dt/instant (:time-millis event))
+            :logger/name  (:logger-name event)
+            :logger/level (str/lower (:level event))}
+
+           (when-let [thrown (:thrown event)]
+             {:trace (:extended-stack-trace thrown)})
+
+           (:context-map event))
+    (do
+      (l/warn :hint "invalid event" :event event)
+      nil)))

backend/src/app/main.clj

@@ -6,39 +6,58 @@
 
 (ns app.main
   (:require
+   [app.common.logging :as l]
    [app.config :as cf]
-   [app.util.logging :as l]
    [app.util.time :as dt]
-   [integrant.core :as ig]))
+   [integrant.core :as ig])
+  (:gen-class))
 
 (def system-config
   {:app.db/pool
    {:uri        (cf/get :database-uri)
     :username   (cf/get :database-username)
     :password   (cf/get :database-password)
+    :read-only  (cf/get :database-readonly false)
     :metrics    (ig/ref :app.metrics/metrics)
     :migrations (ig/ref :app.migrations/all)
     :name :main
-    :min-pool-size 0
-    :max-pool-size 20}
+    :min-size (cf/get :database-min-pool-size 0)
+    :max-size (cf/get :database-max-pool-size 30)}
 
-   :app.metrics/metrics
-   {:definitions
-    {:profile-register
-     {:name "actions_profile_register_count"
-      :help "A global counter of user registrations."
-      :type :counter}
-     :profile-activation
-     {:name "actions_profile_activation_count"
-      :help "A global counter of profile activations"
-      :type :counter}}}
+   ;; Default thread pool for IO operations
+   [::default :app.worker/executor]
+   {:parallelism (cf/get :default-executor-parallelism 60)
+    :prefix :default}
 
-   :app.migrations/all
-   {:main (ig/ref :app.migrations/migrations)}
+   ;; Constrained thread pool. Should only be used from high demand
+   ;; RPC methods.
+   [::blocking :app.worker/executor]
+   {:parallelism (cf/get :blocking-executor-parallelism 20)
+    :prefix :blocking}
+
+   ;; Dedicated thread pool for backround tasks execution.
+   [::worker :app.worker/executor]
+   {:parallelism (cf/get :worker-executor-parallelism 10)
+    :prefix :worker}
+
+   :app.worker/executors
+   {:default  (ig/ref [::default :app.worker/executor])
+    :worker   (ig/ref [::worker :app.worker/executor])
+    :blocking (ig/ref [::blocking :app.worker/executor])}
+
+   :app.worker/executors-monitor
+   {:metrics   (ig/ref :app.metrics/metrics)
+    :executors (ig/ref :app.worker/executors)}
 
    :app.migrations/migrations
    {}
 
+   :app.metrics/metrics
+   {}
+
+   :app.migrations/all
+   {:main (ig/ref :app.migrations/migrations)}
+
    :app.msgbus/msgbus
    {:backend   (cf/get :msgbus-backend :redis)
     :redis-uri (cf/get :redis-uri)}
@@ -54,13 +73,9 @@
    :app.storage/gc-touched-task
    {:pool     (ig/ref :app.db/pool)}
 
-   :app.storage/recheck-task
-   {:pool     (ig/ref :app.db/pool)
-    :storage  (ig/ref :app.storage/storage)}
-
    :app.http.session/session
-   {:pool   (ig/ref :app.db/pool)
-    :tokens (ig/ref :app.tokens/tokens)}
+   {:pool     (ig/ref :app.db/pool)
+    :tokens   (ig/ref :app.tokens/tokens)}
 
    :app.http.session/gc-task
    {:pool        (ig/ref :app.db/pool)
@@ -69,7 +84,7 @@
    :app.http.session/updater
    {:pool           (ig/ref :app.db/pool)
     :metrics        (ig/ref :app.metrics/metrics)
-    :executor       (ig/ref :app.worker/executor)
+    :executor       (ig/ref [::worker :app.worker/executor])
     :session        (ig/ref :app.http.session/session)
     :max-batch-age  (cf/get :http-session-updater-batch-max-age)
     :max-batch-size (cf/get :http-session-updater-batch-max-size)}
@@ -79,59 +94,58 @@
     :pool    (ig/ref :app.db/pool)}
 
    :app.http/server
-   {:port    (cf/get :http-server-port)
-    :router  (ig/ref :app.http/router)
-    :metrics (ig/ref :app.metrics/metrics)
-    :ws      {"/ws/notifications" (ig/ref :app.notifications/handler)}}
+   {:port        (cf/get :http-server-port)
+    :host        (cf/get :http-server-host)
+    :router      (ig/ref :app.http/router)
+    :metrics     (ig/ref :app.metrics/metrics)
+
+    :max-threads (cf/get :http-server-max-threads)
+    :min-threads (cf/get :http-server-min-threads)}
 
    :app.http/router
-   {:rpc         (ig/ref :app.rpc/rpc)
-    :session     (ig/ref :app.http.session/session)
-    :tokens      (ig/ref :app.tokens/tokens)
-    :public-uri  (cf/get :public-uri)
-    :metrics     (ig/ref :app.metrics/metrics)
-    :oauth       (ig/ref :app.http.oauth/handler)
-    :assets      (ig/ref :app.http.assets/handlers)
-    :storage     (ig/ref :app.storage/storage)
-    :sns-webhook (ig/ref :app.http.awsns/handler)
-    :feedback    (ig/ref :app.http.feedback/handler)
-    :error-report-handler (ig/ref :app.loggers.mattermost/handler)}
+   {:assets               (ig/ref :app.http.assets/handlers)
+    :feedback             (ig/ref :app.http.feedback/handler)
+    :session              (ig/ref :app.http.session/session)
+    :sns-webhook          (ig/ref :app.http.awsns/handler)
+    :oauth                (ig/ref :app.http.oauth/handler)
+    :debug                (ig/ref :app.http.debug/handlers)
+    :ws                   (ig/ref :app.http.websocket/handler)
+    :metrics              (ig/ref :app.metrics/metrics)
+    :public-uri           (cf/get :public-uri)
+    :storage              (ig/ref :app.storage/storage)
+    :tokens               (ig/ref :app.tokens/tokens)
+    :audit-http-handler   (ig/ref :app.loggers.audit/http-handler)
+    :rpc                  (ig/ref :app.rpc/rpc)}
+
+   :app.http.debug/handlers
+   {:pool (ig/ref :app.db/pool)
+    :executor (ig/ref [::default :app.worker/executor])}
+
+   :app.http.websocket/handler
+   {:pool     (ig/ref :app.db/pool)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :msgbus   (ig/ref :app.msgbus/msgbus)}
 
    :app.http.assets/handlers
    {:metrics           (ig/ref :app.metrics/metrics)
     :assets-path       (cf/get :assets-path)
     :storage           (ig/ref :app.storage/storage)
+    :executor          (ig/ref [::default :app.worker/executor])
     :cache-max-age     (dt/duration {:hours 24})
     :signature-max-age (dt/duration {:hours 24 :minutes 5})}
 
    :app.http.feedback/handler
-   {:pool (ig/ref :app.db/pool)}
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::default :app.worker/executor])}
 
    :app.http.oauth/handler
-   {:rpc           (ig/ref :app.rpc/rpc)
-    :session       (ig/ref :app.http.session/session)
-    :pool          (ig/ref :app.db/pool)
-    :tokens        (ig/ref :app.tokens/tokens)
-    :audit         (ig/ref :app.loggers.audit/collector)
-    :public-uri    (cf/get :public-uri)}
-
-   ;; RLimit definition for password hashing
-   :app.rlimits/password
-   (cf/get :rlimits-password)
-
-   ;; RLimit definition for image processing
-   :app.rlimits/image
-   (cf/get :rlimits-image)
-
-   ;; RLimit definition for font processing
-   :app.rlimits/font
-   (cf/get :rlimits-font 2)
-
-   ;; A collection of rlimits as hash-map.
-   :app.rlimits/all
-   {:password (ig/ref :app.rlimits/password)
-    :image    (ig/ref :app.rlimits/image)
-    :font     (ig/ref :app.rlimits/font)}
+   {:rpc        (ig/ref :app.rpc/rpc)
+    :session    (ig/ref :app.http.session/session)
+    :pool       (ig/ref :app.db/pool)
+    :tokens     (ig/ref :app.tokens/tokens)
+    :audit      (ig/ref :app.loggers.audit/collector)
+    :executor   (ig/ref [::default :app.worker/executor])
+    :public-uri (cf/get :public-uri)}
 
    :app.rpc/rpc
    {:pool       (ig/ref :app.db/pool)
@@ -140,31 +154,18 @@
     :metrics    (ig/ref :app.metrics/metrics)
     :storage    (ig/ref :app.storage/storage)
     :msgbus     (ig/ref :app.msgbus/msgbus)
-    :rlimits    (ig/ref :app.rlimits/all)
     :public-uri (cf/get :public-uri)
-    :audit      (ig/ref :app.loggers.audit/collector)}
-
-   :app.notifications/handler
-   {:msgbus   (ig/ref :app.msgbus/msgbus)
-    :pool     (ig/ref :app.db/pool)
-    :session  (ig/ref :app.http.session/session)
-    :metrics  (ig/ref :app.metrics/metrics)
-    :executor (ig/ref :app.worker/executor)}
-
-   :app.worker/executor
-   {:min-threads 0
-    :max-threads 256
-    :idle-timeout 60000
-    :name :worker}
+    :audit      (ig/ref :app.loggers.audit/collector)
+    :executors  (ig/ref :app.worker/executors)}
 
    :app.worker/worker
-   {:executor (ig/ref :app.worker/executor)
+   {:executor (ig/ref [::worker :app.worker/executor])
     :tasks    (ig/ref :app.worker/registry)
     :metrics  (ig/ref :app.metrics/metrics)
     :pool     (ig/ref :app.db/pool)}
 
    :app.worker/scheduler
-   {:executor   (ig/ref :app.worker/executor)
+   {:executor   (ig/ref [::worker :app.worker/executor])
     :tasks      (ig/ref :app.worker/registry)
     :pool       (ig/ref :app.db/pool)
     :schedule
@@ -183,9 +184,6 @@
      {:cron #app/cron "0 0 0 * * ?"  ;; daily
       :task :session-gc}
 
-     {:cron #app/cron "0 0 * * * ?"  ;; hourly
-      :task :storage-recheck}
-
      {:cron #app/cron "0 0 0 * * ?"  ;; daily
       :task :objects-gc}
 
@@ -196,16 +194,17 @@
        {:cron #app/cron "0 0 * * * ?"  ;; hourly
         :task :file-offload})
 
-     (when (cf/get :audit-archive-enabled)
-       {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive})
+     (when (contains? cf/flags :audit-log-archive)
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
+        :task :audit-log-archive})
 
-     (when (cf/get :audit-archive-gc-enabled)
-       {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive-gc})
+     (when (contains? cf/flags :audit-log-gc)
+       {:cron #app/cron "0 0 0 * * ?" ;; daily
+        :task :audit-log-gc})
 
-     (when (cf/get :telemetry-enabled)
-       {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
+     (when (or (contains? cf/flags :telemetry)
+               (cf/get :telemetry-enabled))
+       {:cron #app/cron "0 30 */3,23 * * ?"
         :task :telemetry})]}
 
    :app.worker/registry
@@ -213,26 +212,22 @@
     :tasks
     {:sendmail           (ig/ref :app.emails/sendmail-handler)
      :objects-gc         (ig/ref :app.tasks.objects-gc/handler)
-     :delete-object      (ig/ref :app.tasks.delete-object/handler)
-     :delete-profile     (ig/ref :app.tasks.delete-profile/handler)
      :file-media-gc      (ig/ref :app.tasks.file-media-gc/handler)
      :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
      :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
      :storage-touched-gc (ig/ref :app.storage/gc-touched-task)
-     :storage-recheck    (ig/ref :app.storage/recheck-task)
      :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
      :telemetry          (ig/ref :app.tasks.telemetry/handler)
      :session-gc         (ig/ref :app.http.session/gc-task)
      :file-offload       (ig/ref :app.tasks.file-offload/handler)
-     :audit-archive      (ig/ref :app.loggers.audit/archive-task)
-     :audit-archive-gc   (ig/ref :app.loggers.audit/archive-gc-task)}}
+     :audit-log-archive  (ig/ref :app.loggers.audit/archive-task)
+     :audit-log-gc       (ig/ref :app.loggers.audit/gc-task)}}
 
    :app.emails/sendmail-handler
    {:host             (cf/get :smtp-host)
     :port             (cf/get :smtp-port)
     :ssl              (cf/get :smtp-ssl)
     :tls              (cf/get :smtp-tls)
-    :enabled          (cf/get :smtp-enabled)
     :username         (cf/get :smtp-username)
     :password         (cf/get :smtp-password)
     :metrics          (ig/ref :app.metrics/metrics)
@@ -243,18 +238,11 @@
    {:pool    (ig/ref :app.db/pool)
     :max-age cf/deletion-delay}
 
-   :app.tasks.delete-object/handler
-   {:pool    (ig/ref :app.db/pool)
-    :storage (ig/ref :app.storage/storage)}
-
    :app.tasks.objects-gc/handler
    {:pool    (ig/ref :app.db/pool)
     :storage (ig/ref :app.storage/storage)
     :max-age cf/deletion-delay}
 
-   :app.tasks.delete-profile/handler
-   {:pool    (ig/ref :app.db/pool)}
-
    :app.tasks.file-media-gc/handler
    {:pool    (ig/ref :app.db/pool)
     :max-age cf/deletion-delay}
@@ -289,59 +277,63 @@
    :app.loggers.zmq/receiver
    {:endpoint (cf/get :loggers-zmq-uri)}
 
+   :app.loggers.audit/http-handler
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])}
+
    :app.loggers.audit/collector
-   {:enabled  (cf/get :audit-enabled false)
-    :pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)}
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.loggers.audit/archive-task
-   {:uri      (cf/get :audit-archive-uri)
-    :enabled  (cf/get :audit-archive-enabled false)
+   {:uri      (cf/get :audit-log-archive-uri)
     :tokens   (ig/ref :app.tokens/tokens)
     :pool     (ig/ref :app.db/pool)}
 
-   :app.loggers.audit/archive-gc-task
-   {:enabled  (cf/get :audit-archive-gc-enabled false)
-    :max-age  (cf/get :audit-archive-gc-max-age cf/deletion-delay)
+   :app.loggers.audit/gc-task
+   {:max-age  (cf/get :audit-log-gc-max-age cf/deletion-delay)
     :pool     (ig/ref :app.db/pool)}
 
    :app.loggers.loki/reporter
    {:uri      (cf/get :loggers-loki-uri)
     :receiver (ig/ref :app.loggers.zmq/receiver)
-    :executor (ig/ref :app.worker/executor)}
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.loggers.mattermost/reporter
    {:uri      (cf/get :error-report-webhook)
     :receiver (ig/ref :app.loggers.zmq/receiver)
     :pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)}
+    :executor (ig/ref [::worker :app.worker/executor])}
 
-   :app.loggers.mattermost/handler
-   {:pool (ig/ref :app.db/pool)}
+   :app.loggers.database/reporter
+   {:receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.storage/storage
    {:pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)
+    :backends
+    {:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
+     :assets-db (ig/ref [::assets :app.storage.db/backend])
+     :assets-fs (ig/ref [::assets :app.storage.fs/backend])
 
-    :backends {
-               :assets-s3 (ig/ref [::assets :app.storage.s3/backend])
-               :assets-db (ig/ref [::assets :app.storage.db/backend])
-               :assets-fs (ig/ref [::assets :app.storage.fs/backend])
-               :tmp       (ig/ref [::tmp  :app.storage.fs/backend])
-               :fdata-s3  (ig/ref [::fdata :app.storage.s3/backend])
+     :tmp       (ig/ref [::tmp  :app.storage.fs/backend])
+     :fdata-s3  (ig/ref [::fdata :app.storage.s3/backend])
 
-               ;; keep this for backward compatibility
-               :s3        (ig/ref [::assets :app.storage.s3/backend])
-               :fs        (ig/ref [::assets :app.storage.fs/backend])}}
+     ;; keep this for backward compatibility
+     :s3        (ig/ref [::assets :app.storage.s3/backend])
+     :fs        (ig/ref [::assets :app.storage.fs/backend])}}
 
    [::fdata :app.storage.s3/backend]
-   {:region (cf/get :storage-fdata-s3-region)
-    :bucket (cf/get :storage-fdata-s3-bucket)
-    :prefix (cf/get :storage-fdata-s3-prefix)}
+   {:region   (cf/get :storage-fdata-s3-region)
+    :bucket   (cf/get :storage-fdata-s3-bucket)
+    :endpoint (cf/get :storage-fdata-s3-endpoint)
+    :prefix   (cf/get :storage-fdata-s3-prefix)}
 
    [::assets :app.storage.s3/backend]
-   {:region (cf/get :storage-assets-s3-region)
-    :bucket (cf/get :storage-assets-s3-bucket)}
+   {:region   (cf/get :storage-assets-s3-region)
+    :endpoint (cf/get :storage-assets-s3-endpoint)
+    :bucket   (cf/get :storage-assets-s3-bucket)}
 
    [::assets :app.storage.fs/backend]
    {:directory (cf/get :storage-assets-fs-directory)}

backend/src/app/media.clj

@@ -12,8 +12,7 @@
    [app.common.media :as cm]
    [app.common.spec :as us]
    [app.config :as cf]
-   [app.rlimits :as rlm]
-   [app.rpc.queries.svg :as svg]
+   [app.util.svg :as svg]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
    [clojure.java.io :as io]
@@ -51,7 +50,6 @@
                :code :media-type-not-allowed
                :hint "Seems like you are uploading an invalid media object"))))
 
-
 (defmulti process :cmd)
 (defmulti process-error class)
 
@@ -66,17 +64,11 @@
   (throw error))
 
 (defn run
-  [{:keys [rlimits] :as cfg} {:keys [rlimit] :or {rlimit :image} :as params}]
-  (us/assert map? rlimits)
-  (let [rlimit (get rlimits rlimit)]
-    (when-not rlimit
-      (ex/raise :type :internal
-                :code :rlimit-not-configured
-                :hint ":image rlimit not configured"))
-    (try
-      (rlm/execute rlimit (process params))
-      (catch Throwable e
-        (process-error e)))))
+  [params]
+  (try
+    (process params)
+    (catch Throwable e
+      (process-error e))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; --- Thumbnails Generation
@@ -180,7 +172,7 @@
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp svg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file
@@ -210,11 +202,9 @@
             :cause error))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; --- Fonts Generation
+;; Fonts Generation
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(def all-fotmats #{"font/woff2", "font/woff", "font/otf", "font/ttf"})
-
 (defmethod process :generate-fonts
   [{:keys [input] :as params}]
   (letfn [(ttf->otf [data]
@@ -334,10 +324,12 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn configure-assets-storage
-  "Given storage map, returns a storage configured with the apropriate
+  "Given storage map, returns a storage configured with the appropriate
   backend for assets."
-  [storage conn]
-  (-> storage
-      (assoc :conn conn)
-      (assoc :backend (cf/get :assets-storage-backend :assets-fs))))
+  ([storage]
+   (assoc storage :backend (cf/get :assets-storage-backend :assets-fs)))
+  ([storage conn]
+   (-> storage
+       (assoc :conn conn)
+       (assoc :backend (cf/get :assets-storage-backend :assets-fs)))))
 

backend/src/app/metrics.clj

@@ -5,48 +5,132 @@
 ;; Copyright (c) UXBOX Labs SL
 
 (ns app.metrics
+  (:refer-clojure :exclude [run!])
   (:require
-   [app.common.exceptions :as ex]
-   [app.util.logging :as l]
+   [app.common.logging :as l]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig])
   (:import
    io.prometheus.client.CollectorRegistry
    io.prometheus.client.Counter
+   io.prometheus.client.Counter$Child
    io.prometheus.client.Gauge
+   io.prometheus.client.Gauge$Child
    io.prometheus.client.Summary
+   io.prometheus.client.Summary$Child
+   io.prometheus.client.Summary$Builder
    io.prometheus.client.Histogram
+   io.prometheus.client.Histogram$Child
    io.prometheus.client.exporter.common.TextFormat
    io.prometheus.client.hotspot.DefaultExports
    io.prometheus.client.jetty.JettyStatisticsCollector
    org.eclipse.jetty.server.handler.StatisticsHandler
    java.io.StringWriter))
 
-(declare instrument-vars!)
-(declare instrument)
+(set! *warn-on-reflection* true)
+
 (declare create-registry)
 (declare create)
+(declare handler)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Entry Point
+;; METRICS SERVICE PROVIDER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- handler
-  [registry _request]
-  (let [samples  (.metricFamilySamples ^CollectorRegistry registry)
-        writer   (StringWriter.)]
-    (TextFormat/write004 writer samples)
-    {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
-     :body (.toString writer)}))
+(def default-metrics
+  {:update-file-changes
+   {:name "rpc_update_file_changes_total"
+    :help "A total number of changes submitted to update-file."
+    :type :counter}
 
-(s/def ::definitions
-  (s/map-of keyword? map?))
+   :update-file-bytes-processed
+   {:name "rpc_update_file_bytes_processed_total"
+    :help "A total number of bytes processed by update-file."
+    :type :counter}
 
-(defmethod ig/pre-init-spec ::metrics [_]
-  (s/keys :opt-un [::definitions]))
+   :rpc-mutation-timing
+   {:name "rpc_mutation_timing"
+    :help "RPC mutation method call timming."
+    :labels ["name"]
+    :type :histogram}
+
+   :rpc-query-timing
+   {:name "rpc_query_timing"
+    :help "RPC query method call timing."
+    :labels ["name"]
+    :type :histogram}
+
+   :websocket-active-connections
+   {:name "websocket_active_connections"
+    :help "Active websocket connections gauge"
+    :type :gauge}
+
+   :websocket-messages-total
+   {:name "websocket_message_total"
+    :help "Counter of processed messages."
+    :labels ["op"]
+    :type :counter}
+
+   :websocket-session-timing
+   {:name "websocket_session_timing"
+    :help "Websocket session timing (seconds)."
+    :type :summary}
+
+   :session-update-total
+   {:name "http_session_update_total"
+    :help "A counter of session update batch events."
+    :type :counter}
+
+   :tasks-timing
+   {:name "penpot_tasks_timing"
+    :help "Background tasks timing (milliseconds)."
+    :labels ["name"]
+    :type :summary}
+
+   :rlimit-queued-submissions
+   {:name "penpot_rlimit_queued_submissions"
+    :help "Current number of queued submissions on RLIMIT."
+    :labels ["name"]
+    :type :gauge}
+
+   :rlimit-used-permits
+   {:name "penpot_rlimit_used_permits"
+    :help "Current number of used permits on RLIMIT."
+    :labels ["name"]
+    :type :gauge}
+
+   :rlimit-acquires-total
+   {:name "penpot_rlimit_acquires_total"
+    :help "Total number of acquire operations on RLIMIT."
+    :labels ["name"]
+    :type :counter}
+
+   :executors-active-threads
+   {:name "penpot_executors_active_threads"
+    :help "Current number of threads available in the executor service."
+    :labels ["name"]
+    :type :gauge}
+
+   :executors-completed-tasks
+   {:name "penpot_executors_completed_tasks_total"
+    :help "Aproximate number of completed tasks by the executor."
+    :labels ["name"]
+    :type :counter}
+
+   :executors-running-threads
+   {:name "penpot_executors_running_threads"
+    :help "Current number of threads with state RUNNING."
+    :labels ["name"]
+    :type :gauge}
+
+   :executors-queued-submissions
+   {:name "penpot_executors_queued_submissions"
+    :help "Current number of queued submissions."
+    :labels ["name"]
+    :type :gauge}})
 
 (defmethod ig/init-key ::metrics
-  [_ {:keys [definitions] :as cfg}]
+  [_ _]
   (l/info :action "initialize metrics")
   (let [registry    (create-registry)
         definitions (reduce-kv (fn [res k v]
@@ -54,7 +138,7 @@
                                       (create)
                                       (assoc res k)))
                                {}
-                               definitions)]
+                               default-metrics)]
     {:handler (partial handler registry)
      :definitions definitions
      :registry registry}))
@@ -64,24 +148,45 @@
 (s/def ::metrics
   (s/keys :req-un [::registry ::handler]))
 
+(defn- handler
+  [registry _ respond _]
+  (let [samples  (.metricFamilySamples ^CollectorRegistry registry)
+        writer   (StringWriter.)]
+    (TextFormat/write004 writer samples)
+    (respond {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
+              :body (.toString writer)})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Implementation
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(def default-empty-labels (into-array String []))
+
+(def default-quantiles
+  [[0.5  0.01]
+   [0.90 0.01]
+   [0.99 0.001]])
+
+(def default-histogram-buckets
+  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+
+(defn run!
+  [{:keys [definitions]} {:keys [id] :as params}]
+  (when-let [mobj (get definitions id)]
+    ((::fn mobj) params)
+    true))
+
 (defn create-registry
   []
   (let [registry (CollectorRegistry.)]
     (DefaultExports/register registry)
     registry))
 
-(defmacro with-measure
-  [& {:keys [expr cb]}]
-  `(let [start# (System/nanoTime)
-         tdown# ~cb]
-     (try
-       ~expr
-       (finally
-         (tdown# (/ (- (System/nanoTime) start#) 1000000))))))
+(defn- is-array?
+  [o]
+  (let [oc (class o)]
+    (and (.isArray ^Class oc)
+         (= (.getComponentType oc) String))))
 
 (defn make-counter
   [{:keys [name help registry reg labels] :as props}]
@@ -92,18 +197,11 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (.inc ^Counter instance))
-
-      (invoke [_ cmd labels]
-        (.. ^Counter instance
-            (labels (into-array String labels))
-            (inc))))))
+    {::instance instance
+     ::fn (fn [{:keys [inc labels] :or {inc 1 labels default-empty-labels}}]
+            (let [instance (.labels instance (if (is-array? labels) labels (into-array String labels)))]
+              (.inc ^Counter$Child instance (double inc))))}))
 
 (defn make-gauge
   [{:keys [name help registry reg labels] :as props}]
@@ -114,57 +212,33 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
-
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (case cmd
-          :inc (.inc ^Gauge instance)
-          :dec (.dec ^Gauge instance)))
-
-      (invoke [_ cmd labels]
-        (let [labels (into-array String [labels])]
-          (case cmd
-            :inc (.. ^Gauge instance (labels labels) (inc))
-            :dec (.. ^Gauge instance (labels labels) (dec))))))))
-
-(def default-quantiles
-  [[0.75 0.02]
-   [0.99 0.001]])
+    {::instance instance
+     ::fn (fn [{:keys [inc dec labels val] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Gauge instance (if (is-array? labels) labels (into-array String labels)))]
+              (cond (number? inc) (.inc ^Gauge$Child instance (double inc))
+                    (number? dec) (.dec ^Gauge$Child instance (double dec))
+                    (number? val) (.set ^Gauge$Child instance (double val)))))}))
 
 (defn make-summary
   [{:keys [name help registry reg labels max-age quantiles buckets]
-    :or {max-age 3600 buckets 6 quantiles default-quantiles} :as props}]
+    :or {max-age 3600 buckets 12 quantiles default-quantiles} :as props}]
   (let [registry (or registry reg)
-        instance (doto (Summary/build)
+        builder  (doto (Summary/build)
                    (.name name)
                    (.help help))
         _        (when (seq quantiles)
-                   (.maxAgeSeconds ^Summary instance max-age)
-                   (.ageBuckets ^Summary instance buckets))
+                   (.maxAgeSeconds ^Summary$Builder builder ^long max-age)
+                   (.ageBuckets ^Summary$Builder builder buckets))
         _        (doseq [[q e] quantiles]
-                   (.quantile ^Summary instance q e))
+                   (.quantile ^Summary$Builder builder q e))
         _        (when (seq labels)
-                   (.labelNames instance (into-array String labels)))
-        instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
+                   (.labelNames ^Summary$Builder builder (into-array String labels)))
+        instance (.register ^Summary$Builder builder registry)]
 
-      clojure.lang.IFn
-      (invoke [_ cmd val]
-        (.observe ^Summary instance val))
-
-      (invoke [_ cmd val labels]
-        (.. ^Summary instance
-            (labels (into-array String labels))
-            (observe val))))))
-
-(def default-histogram-buckets
-  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+    {::instance instance
+     ::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Summary instance (if (is-array? labels) labels (into-array String labels)))]
+              (.observe ^Summary$Child instance val)))}))
 
 (defn make-histogram
   [{:keys [name help registry reg labels buckets]
@@ -177,18 +251,11 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd val]
-        (.observe ^Histogram instance val))
-
-      (invoke [_ cmd val labels]
-        (.. ^Histogram instance
-            (labels (into-array String labels))
-            (observe val))))))
+    {::instance instance
+     ::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Histogram instance (if (is-array? labels) labels (into-array String labels)))]
+              (.observe ^Histogram$Child instance val)))}))
 
 (defn create
   [{:keys [type] :as props}]
@@ -198,114 +265,6 @@
     :summary (make-summary props)
     :histogram (make-histogram props)))
 
-(defn wrap-counter
-  ([rootf mobj]
-   (let [mdata (meta rootf)
-         origf (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-          (mobj :inc)
-          (origf a))
-         ([a b]
-          (mobj :inc)
-          (origf a b))
-         ([a b c]
-          (mobj :inc)
-          (origf a b c))
-         ([a b c d]
-          (mobj :inc)
-          (origf a b c d))
-         ([a b c d & more]
-          (mobj :inc)
-          (apply origf a b c d more)))
-       (assoc mdata ::original origf))))
-  ([rootf mobj labels]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-          (mobj :inc labels)
-          (origf a))
-         ([a b]
-          (mobj :inc labels)
-          (origf a b))
-         ([a b & more]
-          (mobj :inc labels)
-          (apply origf a b more)))
-       (assoc mdata ::original origf)))))
-
-(defn wrap-summary
-  ([rootf mobj]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-         (with-measure
-           :expr (origf a)
-           :cb   #(mobj :observe %)))
-         ([a b]
-          (with-measure
-            :expr (origf a b)
-            :cb   #(mobj :observe %)))
-         ([a b & more]
-          (with-measure
-            :expr (apply origf a b more)
-            :cb   #(mobj :observe %))))
-       (assoc mdata ::original origf))))
-
-  ([rootf mobj labels]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-         (with-measure
-           :expr (origf a)
-           :cb   #(mobj :observe % labels)))
-         ([a b]
-          (with-measure
-            :expr (origf a b)
-            :cb   #(mobj :observe % labels)))
-         ([a b & more]
-          (with-measure
-            :expr (apply origf a b more)
-            :cb   #(mobj :observe % labels))))
-       (assoc mdata ::original origf)))))
-
-(defn instrument-vars!
-  [vars {:keys [wrap] :as props}]
-  (let [obj (create props)]
-    (cond
-      (instance? Counter @obj)
-      (doseq [var vars]
-        (alter-var-root var (or wrap wrap-counter) obj))
-
-      (instance? Summary @obj)
-      (doseq [var vars]
-        (alter-var-root var (or wrap wrap-summary) obj))
-
-      :else
-      (ex/raise :type :not-implemented))))
-
-(defn instrument
-  [f {:keys [wrap] :as props}]
-  (let [obj (create props)]
-    (cond
-      (instance? Counter @obj)
-      ((or wrap wrap-counter) f obj)
-
-      (instance? Summary @obj)
-      ((or wrap wrap-summary) f obj)
-
-      (instance? Histogram @obj)
-      ((or wrap wrap-summary) f obj)
-
-      :else
-      (ex/raise :type :not-implemented))))
-
 (defn instrument-jetty!
   [^CollectorRegistry registry ^StatisticsHandler handler]
   (doto (JettyStatisticsCollector. handler)

backend/src/app/migrations.clj

@@ -193,6 +193,21 @@
 
    {:name "0061-mod-file-table"
     :fn (mg/resource "app/migrations/sql/0061-mod-file-table.sql")}
+
+   {:name "0062-fix-metadata-media"
+    :fn (mg/resource "app/migrations/sql/0062-fix-metadata-media.sql")}
+
+   {:name "0063-add-share-link-table"
+    :fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")}
+
+   {:name "0064-mod-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")}
+
+   {:name "0065-add-trivial-spelling-fixes"
+    :fn (mg/resource "app/migrations/sql/0065-add-trivial-spelling-fixes.sql")}
+   
+   {:name "0066-add-frame-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0066-add-frame-thumbnail-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0035-add-storage-tables.sql

@@ -22,7 +22,7 @@ CREATE TABLE storage_data (
 CREATE INDEX storage_data__id__idx ON storage_data(id);
 
 -- Table used for store inflight upload ids, for later recheck and
--- delete possible staled files that exists on the phisical storage
+-- delete possible staled files that exists on the physical storage
 -- but does not exists in the 'storage_object' table.
 
 CREATE TABLE storage_pending (

backend/src/app/migrations/sql/0062-fix-metadata-media.sql

@@ -1,4 +1,4 @@
--- Fix problem with content-type inconherence
+-- Fix problem with content-type incoherence
 
 UPDATE storage_object so
 SET metadata = jsonb_set(metadata, '{~:content-type}', to_jsonb(fmo.mtype))

backend/src/app/migrations/sql/0063-add-share-link-table.sql

@@ -0,0 +1,12 @@
+CREATE TABLE share_link (
+  id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
+  owner_id uuid NULL REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE,
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  pages uuid[],
+  flags text[]
+);
+
+CREATE INDEX share_link_file_id_idx ON share_link(file_id);
+CREATE INDEX share_link_owner_id_idx ON share_link(owner_id);

backend/src/app/migrations/sql/0064-mod-audit-log-table.sql

@@ -0,0 +1,13 @@
+ALTER TABLE audit_log
+  ADD COLUMN tracked_at timestamptz NULL DEFAULT clock_timestamp(),
+  ADD COLUMN source text NULL,
+  ADD COLUMN context jsonb NULL;
+
+ALTER TABLE audit_log
+  ALTER COLUMN source SET STORAGE external,
+  ALTER COLUMN context SET STORAGE external;
+
+UPDATE audit_log SET source = 'backend', tracked_at=created_at;
+
+-- ALTER TABLE audit_log ALTER COLUMN source SET NOT NULL;
+-- ALTER TABLE audit_log ALTER COLUMN tracked_at SET NOT NULL;

backend/src/app/migrations/sql/0065-add-trivial-spelling-fixes.sql

@@ -0,0 +1,2 @@
+ALTER INDEX file__modified_at__has_media_trimed__idx RENAME TO file__modified_at__has_media_trimmed__idx;
+ALTER INDEX media_bject__file_id__idx RENAME TO media_object__file_id__idx;

backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql

@@ -0,0 +1,10 @@
+CREATE TABLE file_frame_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
+  frame_id uuid NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  data text NULL,
+
+  PRIMARY KEY(file_id, frame_id)
+);

backend/src/app/migrations/sql/XXXX-drop-obsolete-tables.sql

@@ -1,5 +1,5 @@
 --- This is a second migration but it should be applied when manual
---- migration intervention is alteady executed.
+--- migration intervention is already executed.
 
 ALTER TABLE file_media_object ALTER COLUMN media_id SET NOT NULL;
 DROP TABLE file_media_thumbnail;

backend/src/app/msgbus.clj

@@ -8,17 +8,16 @@
   "The msgbus abstraction implemented using redis as underlying backend."
   (:require
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cfg]
    [app.util.blob :as blob]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.core :as p])
   (:import
-   java.time.Duration
    io.lettuce.core.RedisClient
    io.lettuce.core.RedisURI
    io.lettuce.core.api.StatefulConnection
@@ -29,7 +28,10 @@
    io.lettuce.core.codec.StringCodec
    io.lettuce.core.pubsub.RedisPubSubListener
    io.lettuce.core.pubsub.StatefulRedisPubSubConnection
-   io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands))
+   io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands
+   io.lettuce.core.resource.ClientResources
+   io.lettuce.core.resource.DefaultClientResources
+   java.time.Duration))
 
 (def ^:private prefix (cfg/get :tenant))
 
@@ -136,27 +138,35 @@
 (declare impl-redis-sub)
 (declare impl-redis-unsub)
 
+
 (defmethod init-backend :redis
   [{:keys [redis-uri] :as cfg}]
   (let [codec    (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE)
 
-        uri      (RedisURI/create redis-uri)
-        rclient  (RedisClient/create ^RedisURI uri)
+        resources (.. (DefaultClientResources/builder)
+                      (ioThreadPoolSize 4)
+                      (computationThreadPoolSize 4)
+                      (build))
 
-        pub-conn (.connect ^RedisClient rclient ^RedisCodec codec)
-        sub-conn (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
+        uri       (RedisURI/create redis-uri)
+        rclient   (RedisClient/create ^ClientResources resources ^RedisURI uri)
+
+        pub-conn  (.connect ^RedisClient rclient ^RedisCodec codec)
+        sub-conn  (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
 
     (.setTimeout ^StatefulRedisConnection pub-conn ^Duration (dt/duration {:seconds 10}))
     (.setTimeout ^StatefulRedisPubSubConnection sub-conn ^Duration (dt/duration {:seconds 10}))
 
     (-> cfg
+        (assoc ::resources resources)
         (assoc ::pub-conn pub-conn)
         (assoc ::sub-conn sub-conn))))
 
 (defmethod stop-backend :redis
-  [{:keys [::pub-conn ::sub-conn] :as cfg}]
+  [{:keys [::pub-conn ::sub-conn ::resources] :as cfg}]
   (.close ^StatefulRedisConnection pub-conn)
-  (.close ^StatefulRedisPubSubConnection sub-conn))
+  (.close ^StatefulRedisPubSubConnection sub-conn)
+  (.shutdown ^ClientResources resources))
 
 (defmethod init-pub-loop :redis
   [{:keys [::pub-conn ::pub-ch]}]
@@ -179,18 +189,18 @@
     ;; Add a unique listener to connection
     (.addListener sub-conn
                   (reify RedisPubSubListener
-                    (message [it pattern topic message])
-                    (message [it topic message]
+                    (message [_ _pattern _topic _message])
+                    (message [_ topic message]
                       ;; There are no back pressure, so we use a slidding
                       ;; buffer for cases when the pubsub broker sends
                       ;; more messages that we can process.
                       (let [val {:topic topic :message (blob/decode message)}]
                         (when-not (a/offer! rcv-ch val)
                           (l/warn :msg "dropping message on subscription loop"))))
-                    (psubscribed [it pattern count])
-                    (punsubscribed [it pattern count])
-                    (subscribed [it topic count])
-                    (unsubscribed [it topic count])))
+                    (psubscribed [_ _pattern _count])
+                    (punsubscribed [_ _pattern _count])
+                    (subscribed [_ _topic _count])
+                    (unsubscribed [_ _topic _count])))
 
     (letfn [(subscribe-to-single-topic [nsubs topic chan]
               (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
@@ -243,7 +253,7 @@
             (recur))
           (a/close! rcv-ch)))
 
-      ;; Asyncrhonous message processing loop;x
+      ;; Asynchronous message processing loop;x
       (a/go-loop []
         (if-let [{:keys [topic message]} (a/<! rcv-ch)]
           ;; This means we receive data from redis and we need to

backend/src/app/notifications.clj

@@ -1,280 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.notifications
-  "A websocket based notifications mechanism."
-  (:require
-   [app.common.spec :as us]
-   [app.common.transit :as t]
-   [app.db :as db]
-   [app.metrics :as mtx]
-   [app.util.async :as aa]
-   [app.util.logging :as l]
-   [app.util.time :as dt]
-   [app.worker :as wrk]
-   [clojure.core.async :as a]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig]
-   [ring.adapter.jetty9 :as jetty]
-   [ring.middleware.cookies :refer [wrap-cookies]]
-   [ring.middleware.keyword-params :refer [wrap-keyword-params]]
-   [ring.middleware.params :refer [wrap-params]]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Handler
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(declare retrieve-file)
-(declare websocket)
-(declare handler)
-
-(s/def ::session map?)
-(s/def ::msgbus fn?)
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::msgbus ::db/pool ::session ::mtx/metrics ::wrk/executor]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [session metrics] :as cfg}]
-  (let [wrap-session    (:middleware session)
-
-        mtx-active-connections
-        (mtx/create
-         {:name "websocket_active_connections"
-          :registry (:registry metrics)
-          :type :gauge
-          :help "Active websocket connections."})
-
-        mtx-messages
-        (mtx/create
-         {:name "websocket_message_total"
-          :registry (:registry metrics)
-          :labels ["op"]
-          :type :counter
-          :help "Counter of processed messages."})
-
-        mtx-sessions
-        (mtx/create
-         {:name "websocket_session_timing"
-          :registry (:registry metrics)
-          :quantiles []
-          :help "Websocket session timing (seconds)."
-          :type :summary})
-
-        cfg (assoc cfg
-                   :mtx-active-connections mtx-active-connections
-                   :mtx-messages mtx-messages
-                   :mtx-sessions mtx-sessions
-                   )]
-    (-> #(handler cfg %)
-        (wrap-session)
-        (wrap-keyword-params)
-        (wrap-cookies)
-        (wrap-params))))
-
-(s/def ::file-id ::us/uuid)
-(s/def ::session-id ::us/uuid)
-
-(s/def ::websocket-handler-params
-  (s/keys :req-un [::file-id ::session-id]))
-
-(defn- handler
-  [{:keys [pool] :as cfg} {:keys [profile-id params] :as req}]
-  (let [params (us/conform ::websocket-handler-params params)
-        file   (retrieve-file pool (:file-id params))
-        cfg    (merge cfg params
-                      {:profile-id profile-id
-                       :team-id (:team-id file)})]
-    (cond
-      (not profile-id)
-      {:error {:code 403 :message "Authentication required"}}
-
-      (not file)
-      {:error {:code 404 :message "File does not exists"}}
-
-      :else
-      (websocket cfg))))
-
-(def ^:private
-  sql:retrieve-file
-  "select f.id as id,
-          p.team_id as team_id
-     from file as f
-     join project as p on (p.id = f.project_id)
-    where f.id = ?")
-
-(defn- retrieve-file
-  [conn id]
-  (db/exec-one! conn [sql:retrieve-file id]))
-
-
-;; --- WEBSOCKET INIT
-
-(declare handle-connect)
-
-(defn- ws-send
-  [conn data]
-  (try
-    (when (jetty/connected? conn)
-      (jetty/send! conn data)
-      true)
-    (catch java.lang.NullPointerException _e
-      false)))
-
-(defn websocket
-  [{:keys [file-id team-id msgbus executor] :as cfg}]
-  (let [rcv-ch       (a/chan 32)
-        out-ch       (a/chan 32)
-        mtx-aconn    (:mtx-active-connections cfg)
-        mtx-messages (:mtx-messages cfg)
-        mtx-sessions (:mtx-sessions cfg)
-        created-at   (dt/now)
-        ws-send      (mtx/wrap-counter ws-send mtx-messages ["send"])]
-
-    (letfn [(on-connect [conn]
-              (mtx-aconn :inc)
-              ;; A subscription channel should use a lossy buffer
-              ;; because we can't penalize normal clients when one
-              ;; slow client is connected to the room.
-              (let [sub-ch (a/chan (a/dropping-buffer 128))
-                    cfg    (assoc cfg
-                                  :conn conn
-                                  :rcv-ch rcv-ch
-                                  :out-ch out-ch
-                                  :sub-ch sub-ch)]
-
-                (l/trace :event "connect" :session (:session-id cfg))
-
-                ;; Forward all messages from out-ch to the websocket
-                ;; connection
-                (a/go-loop []
-                  (let [val (a/<! out-ch)]
-                    (when (some? val)
-                      (when (a/<! (aa/thread-call executor #(ws-send conn (t/encode-str val))))
-                        (recur)))))
-
-                (a/go
-                  ;; Subscribe to corresponding topics
-                  (a/<! (msgbus :sub {:topics [file-id team-id] :chan sub-ch}))
-                  (a/<! (handle-connect cfg))
-
-                  ;; when connection is closed
-                  (mtx-aconn :dec)
-                  (mtx-sessions :observe (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0))
-
-                  ;; close subscription
-                  (a/close! sub-ch))))
-
-            (on-error [_conn _e]
-              (l/trace :event "error" :session (:session-id cfg))
-
-              (a/close! out-ch)
-              (a/close! rcv-ch))
-
-            (on-close [_conn _status _reason]
-              (l/trace :event "close" :session (:session-id cfg))
-
-              (a/close! out-ch)
-              (a/close! rcv-ch))
-
-            (on-message [_ws message]
-              (let [message (t/decode-str message)]
-                (when-not (a/offer! rcv-ch message)
-                  (l/warn :msg "drop messages"))))]
-
-      {:on-connect on-connect
-       :on-error on-error
-       :on-close on-close
-       :on-text (mtx/wrap-counter on-message mtx-messages ["recv"])
-       :on-bytes (constantly nil)})))
-
-;; --- CONNECTION INIT
-
-(declare send-presence)
-(declare handle-message)
-(declare start-loop!)
-
-(defn- handle-connect
-  [cfg]
-  (a/go
-    (a/<! (handle-message cfg {:type :connect}))
-    (a/<! (start-loop! cfg))
-    (a/<! (handle-message cfg {:type :disconnect}))))
-
-(defn- start-loop!
-  [{:keys [rcv-ch out-ch sub-ch session-id] :as cfg}]
-  (a/go-loop []
-    (let [timeout    (a/timeout 30000)
-          [val port] (a/alts! [rcv-ch sub-ch timeout])]
-      (cond
-        ;; Process message coming from connected client
-        (and (= port rcv-ch) (some? val))
-        (do
-          (a/<! (handle-message cfg val))
-          (recur))
-
-        ;; Process message coming from pubsub.
-        (and (= port sub-ch) (some? val))
-        (do
-          (when-not (= (:session-id val) session-id)
-            ;; If we receive a connect message of other user, we need
-            ;; to send an update presence to all participants.
-            (when (= :connect (:type val))
-              (a/<! (send-presence cfg :presence)))
-
-            ;; Then, just forward the message
-            (a/>! out-ch val))
-          (recur))
-
-        ;; When timeout channel is signaled, we need to send a ping
-        ;; message to the output channel. TODO: we need to make this
-        ;; more smart.
-        (= port timeout)
-        (do
-          (a/>! out-ch {:type :ping})
-          (recur))))))
-
-(defn send-presence
-  ([cfg] (send-presence cfg :presence))
-  ([{:keys [msgbus session-id profile-id file-id]} type]
-   (a/go
-     (a/<! (msgbus :pub {:topic file-id
-                         :message {:type type
-                                   :session-id session-id
-                                   :profile-id profile-id}})))))
-
-;; --- INCOMING MSG PROCESSING
-
-(defmulti handle-message
-  (fn [_ message] (:type message)))
-
-(defmethod handle-message :connect
-  [cfg _]
-  (send-presence cfg :connect))
-
-(defmethod handle-message :disconnect
-  [cfg _]
-  (send-presence cfg :disconnect))
-
-(defmethod handle-message :keepalive
-  [_ _]
-  (a/go :nothing))
-
-(defmethod handle-message :pointer-update
-  [{:keys [profile-id file-id session-id msgbus] :as cfg} message]
-  (let [message (assoc message
-                       :profile-id profile-id
-                       :session-id session-id)]
-    (msgbus :pub {:topic file-id
-                  :message message})))
-
-(defmethod handle-message :default
-  [_ws message]
-  (a/go
-    (l/log :level :warn
-           :msg "received unexpected message"
-           :message message)))
-

backend/src/app/rlimits.clj

@@ -1,45 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.rlimits
-  "Resource usage limits (in other words: semaphores)."
-  (:require
-   [app.common.spec :as us]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig])
-  (:import
-   java.util.concurrent.Semaphore))
-
-(s/def ::rlimit  #(instance? Semaphore %))
-(s/def ::rlimits (s/map-of ::us/keyword ::rlimit))
-
-(derive ::password ::instance)
-(derive ::image ::instance)
-(derive ::font ::instance)
-
-(defmethod ig/pre-init-spec ::instance [_]
-  (s/spec int?))
-
-(defmethod ig/init-key ::instance
-  [_ permits]
-  (Semaphore. (int permits)))
-
-(defn acquire!
-  [sem]
-  (.acquire ^Semaphore sem))
-
-(defn release!
-  [sem]
-  (.release ^Semaphore sem))
-
-(defmacro execute
-  [rlinst & body]
-  `(try
-     (acquire! ~rlinst)
-     ~@body
-     (finally
-       (release! ~rlinst))))
-

backend/src/app/rpc.clj

@@ -8,173 +8,211 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.metrics :as mtx]
-   [app.rlimits :as rlm]
-   [app.util.logging :as l]
+   [app.rpc.retry :as retry]
+   [app.rpc.rlimit :as rlimit]
+   [app.util.async :as async]
    [app.util.services :as sv]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]))
 
 (defn- default-handler
   [_]
-  (ex/raise :type :not-found))
+  (p/rejected (ex/error :type :not-found)))
 
-(defn- run-hook
-  [hook-fn response]
-  (ex/ignoring (hook-fn))
+(defn- handle-response-transformation
+  [response request mdata]
+  (if-let [transform-fn (:transform-response mdata)]
+    (transform-fn request response)
+    response))
+
+(defn- handle-before-comple-hook
+  [response mdata]
+  (when-let [hook-fn (:before-complete mdata)]
+    (ex/ignoring (hook-fn)))
   response)
 
 (defn- rpc-query-handler
-  [methods {:keys [profile-id] :as request}]
-  (let [type   (keyword (get-in request [:path-params :type]))
+  "Ring handler that dispatches query requests and convert between
+  internal async flow into ring async flow."
+  [methods {:keys [profile-id session-id] :as request} respond raise]
+  (letfn [(handle-response [result]
+            (let [mdata (meta result)]
+              (-> {:status 200 :body result}
+                  (handle-response-transformation request mdata))))]
 
-        data   (merge (:params request)
-                      (:body-params request)
-                      (:uploads request)
-                      {::request request})
+    (let [type   (keyword (get-in request [:path-params :type]))
+          data   (merge (:params request)
+                        (:body-params request)
+                        (:uploads request)
+                        {::request request})
 
-        data   (if profile-id
-                 (assoc data :profile-id profile-id)
-                 (dissoc data :profile-id))
+          data   (if profile-id
+                   (assoc data :profile-id profile-id ::session-id session-id)
+                   (dissoc data :profile-id))
 
-        result ((get methods type default-handler) data)
-        mdata  (meta result)]
+          ;; Get the method from methods registry and if method does
+          ;; not exists asigns it to the default handler.
+          method (get methods type default-handler)]
 
-    (cond->> {:status 200 :body result}
-      (fn? (:transform-response mdata))
-      ((:transform-response mdata) request))))
+      (-> (method data)
+          (p/then #(respond (handle-response %)))
+          (p/catch raise)))))
 
 (defn- rpc-mutation-handler
-  [methods {:keys [profile-id] :as request}]
-  (let [type   (keyword (get-in request [:path-params :type]))
-        data   (merge (:params request)
-                      (:body-params request)
-                      (:uploads request)
-                      {::request request})
+  "Ring handler that dispatches mutation requests and convert between
+  internal async flow into ring async flow."
+  [methods {:keys [profile-id session-id] :as request} respond raise]
+  (letfn [(handle-response [result]
+            (let [mdata (meta result)]
+              (-> {:status 200 :body result}
+                  (handle-response-transformation request mdata)
+                  (handle-before-comple-hook mdata))))]
 
-        data   (if profile-id
-                 (assoc data :profile-id profile-id)
-                 (dissoc data :profile-id))
+    (let [type   (keyword (get-in request [:path-params :type]))
+          data   (merge (:params request)
+                        (:body-params request)
+                        (:uploads request)
+                        {::request request})
 
-        result ((get methods type default-handler) data)
-        mdata  (meta result)]
-    (cond->> {:status 200 :body result}
-      (fn? (:transform-response mdata))
-      ((:transform-response mdata) request)
+          data   (if profile-id
+                   (assoc data :profile-id profile-id ::session-id session-id)
+                   (dissoc data :profile-id))
 
-      (fn? (:before-complete mdata))
-      (run-hook (:before-complete mdata)))))
+          method (get methods type default-handler)]
 
-(defn- wrap-with-metrics
-  [cfg f mdata]
-  (mtx/wrap-summary f (::mobj cfg) [(::sv/name mdata)]))
+      (-> (method data)
+          (p/then #(respond (handle-response %)))
+          (p/catch raise)))))
 
-;; Wrap the rpc handler with a semaphore if it is specified in the
-;; metadata asocciated with the handler.
-(defn- wrap-with-rlimits
-  [cfg f mdata]
-  (if-let [key (:rlimit mdata)]
-    (let [rlinst (get-in cfg [:rlimits key])]
-      (when-not rlinst
-        (ex/raise :type :internal
-                  :code :rlimit-not-configured
-                  :hint (str/fmt "%s rlimit not configured" key)))
-      (l/trace :action "add rlimit"
-               :handler (::sv/name mdata))
-      (fn [cfg params]
-        (rlm/execute rlinst (f cfg params))))
+(defn- wrap-metrics
+  "Wrap service method with metrics measurement."
+  [{:keys [metrics ::metrics-id]} f mdata]
+  (let [labels (into-array String [(::sv/name mdata)])]
+    (fn [cfg params]
+      (let [start (System/nanoTime)]
+        (p/finally
+          (f cfg params)
+          (fn [_ _]
+            (mtx/run! metrics
+                      {:id metrics-id
+                       :val (/ (- (System/nanoTime) start) 1000000)
+                       :labels labels})))))))
+
+(defn- wrap-dispatch
+  "Wraps service method into async flow, with the ability to dispatching
+  it to a preconfigured executor service."
+  [{:keys [executors] :as cfg} f mdata]
+  (let [dname (::async/dispatch mdata :none)]
+    (if (= :none dname)
+      (with-meta
+        (fn [cfg params]
+          (p/do! (f cfg params)))
+        mdata)
+
+      (let [executor (get executors dname)]
+        (when-not executor
+          (ex/raise :type :internal
+                    :code :executor-not-configured
+                    :hint (format "executor %s not configured" dname)))
+        (with-meta
+          (fn [cfg params]
+            (-> (px/submit! executor #(f cfg params))
+                (p/bind p/wrap)))
+          mdata)))))
+
+(defn- wrap-audit
+  [{:keys [audit] :as cfg} f mdata]
+  (if audit
+    (with-meta
+      (fn [cfg {:keys [::request] :as params}]
+        (p/finally (f cfg params)
+                   (fn [result _]
+                     (when result
+                       (let [resultm    (meta result)
+                             profile-id (or (:profile-id params)
+                                            (:profile-id result)
+                                            (::audit/profile-id resultm))
+                             props      (d/merge params (::audit/props resultm))]
+                         (audit :cmd :submit
+                                :type (or (::audit/type resultm)
+                                          (::type cfg))
+                                :name (or (::audit/name resultm)
+                                          (::sv/name mdata))
+                                :profile-id profile-id
+                                :ip-addr (audit/parse-client-ip request)
+                                :props (dissoc props ::request)))))))
+      mdata)
     f))
 
-(defn- wrap-impl
-  [{:keys [audit] :as cfg} f mdata]
-  (let [f      (wrap-with-rlimits cfg f mdata)
-        f      (wrap-with-metrics cfg f mdata)
-        spec   (or (::sv/spec mdata) (s/spec any?))
-        auth?  (:auth mdata true)]
+(defn- wrap
+  [cfg f mdata]
+  (let [f     (as-> f $
+                (wrap-dispatch cfg $ mdata)
+                (rlimit/wrap-rlimit cfg $ mdata)
+                (retry/wrap-retry cfg $ mdata)
+                (wrap-audit cfg $ mdata)
+                (wrap-metrics cfg $ mdata)
+                )
+
+        spec  (or (::sv/spec mdata) (s/spec any?))
+        auth? (:auth mdata true)]
 
     (l/trace :action "register" :name (::sv/name mdata))
-    (fn [params]
+    (with-meta
+      (fn [{:keys [::request] :as params}]
+        ;; Raise authentication error when rpc method requires auth but
+        ;; no profile-id is found in the request.
+        (p/do!
+         (if (and auth? (not (uuid? (:profile-id params))))
+           (ex/raise :type :authentication
+                     :code :authentication-required
+                     :hint "authentication required for this endpoint")
+           (let [params (us/conform spec (dissoc params ::request))]
+             (f cfg (assoc params ::request request))))))
 
-      ;; Raise authentication error when rpc method requires auth but
-      ;; no profile-id is found in the request.
-      (when (and auth? (not (uuid? (:profile-id params))))
-        (ex/raise :type :authentication
-                  :code :authentication-required
-                  :hint "authentication required for this endpoint"))
-
-      (let [params' (dissoc params ::request)
-            params' (us/conform spec params')
-            result  (f cfg params')]
-
-        ;; When audit log is enabled (default false).
-        (when (fn? audit)
-          (let [resultm    (meta result)
-                request    (::request params)
-                profile-id (or (:profile-id params')
-                               (:profile-id result)
-                               (::audit/profile-id resultm))
-                props      (d/merge params (::audit/props resultm))]
-            (audit :cmd :submit
-                   :type (::type cfg)
-                   :name (or (::audit/name resultm)
-                             (::sv/name mdata))
-                   :profile-id profile-id
-                   :ip-addr (audit/parse-client-ip request)
-                   :props (audit/profile->props props))))
-
-        result))))
+      mdata)))
 
 (defn- process-method
   [cfg vfn]
   (let [mdata (meta vfn)]
     [(keyword (::sv/name mdata))
-     (wrap-impl cfg (deref vfn) mdata)]))
+     (wrap cfg (deref vfn) mdata)]))
 
 (defn- resolve-query-methods
   [cfg]
-  (let [mobj (mtx/create
-              {:name "rpc_query_timing"
-               :labels ["name"]
-               :registry (get-in cfg [:metrics :registry])
-               :type :histogram
-               :help "Timing of query services."})
-        cfg  (assoc cfg ::mobj mobj ::type "query")]
+  (let [cfg (assoc cfg ::type "query" ::metrics-id :rpc-query-timing)]
     (->> (sv/scan-ns 'app.rpc.queries.projects
                      'app.rpc.queries.files
                      'app.rpc.queries.teams
                      'app.rpc.queries.comments
                      'app.rpc.queries.profile
-                     'app.rpc.queries.recent-files
                      'app.rpc.queries.viewer
-                     'app.rpc.queries.fonts
-                     'app.rpc.queries.svg)
+                     'app.rpc.queries.fonts)
          (map (partial process-method cfg))
          (into {}))))
 
 (defn- resolve-mutation-methods
   [cfg]
-  (let [mobj (mtx/create
-              {:name "rpc_mutation_timing"
-               :labels ["name"]
-               :registry (get-in cfg [:metrics :registry])
-               :type :histogram
-               :help "Timing of mutation services."})
-        cfg  (assoc cfg ::mobj mobj ::type "mutation")]
+  (let [cfg (assoc cfg ::type "mutation" ::metrics-id :rpc-mutation-timing)]
     (->> (sv/scan-ns 'app.rpc.mutations.demo
                      'app.rpc.mutations.media
                      'app.rpc.mutations.profile
                      'app.rpc.mutations.files
                      'app.rpc.mutations.comments
                      'app.rpc.mutations.projects
-                     'app.rpc.mutations.viewer
                      'app.rpc.mutations.teams
                      'app.rpc.mutations.management
                      'app.rpc.mutations.ldap
                      'app.rpc.mutations.fonts
+                     'app.rpc.mutations.share-link
                      'app.rpc.mutations.verify-token)
          (map (partial process-method cfg))
          (into {}))))
@@ -183,15 +221,16 @@
 (s/def ::session map?)
 (s/def ::tokens fn?)
 (s/def ::audit (s/nilable fn?))
+(s/def ::executors (s/map-of keyword? ::wrk/executor))
 
 (defmethod ig/pre-init-spec ::rpc [_]
   (s/keys :req-un [::storage ::session ::tokens ::audit
-                   ::mtx/metrics ::rlm/rlimits ::db/pool]))
+                   ::executors ::mtx/metrics ::db/pool]))
 
 (defmethod ig/init-key ::rpc
   [_ cfg]
   (let [mq (resolve-query-methods cfg)
         mm (resolve-mutation-methods cfg)]
     {:methods {:query mq :mutation mm}
-     :query-handler #(rpc-query-handler mq %)
-     :mutation-handler #(rpc-mutation-handler mm %)}))
+     :query-handler (partial rpc-query-handler mq)
+     :mutation-handler (partial rpc-mutation-handler mm)}))

backend/src/app/rpc/mutations/comments.clj

@@ -7,10 +7,12 @@
 (ns app.rpc.mutations.comments
   (:require
    [app.common.exceptions :as ex]
+   [app.common.geom.point :as gpt]
    [app.common.spec :as us]
    [app.db :as db]
    [app.rpc.queries.comments :as comments]
    [app.rpc.queries.files :as files]
+   [app.rpc.retry :as retry]
    [app.util.blob :as blob]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -25,13 +27,15 @@
 (s/def ::page-id ::us/uuid)
 (s/def ::file-id ::us/uuid)
 (s/def ::profile-id ::us/uuid)
-(s/def ::position ::us/point)
+(s/def ::position ::gpt/point)
 (s/def ::content ::us/string)
 
 (s/def ::create-comment-thread
   (s/keys :req-un [::profile-id ::file-id ::position ::content ::page-id]))
 
 (sv/defmethod ::create-comment-thread
+  {::retry/max-retries 3
+   ::retry/matches retry/conflict-db-insert?}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
     (files/check-read-permissions! conn profile-id file-id)
@@ -43,7 +47,7 @@
         res (db/exec-one! conn [sql file-id])]
     (:next-seqn res)))
 
-(defn- create-comment-thread*
+(defn- create-comment-thread
   [conn {:keys [profile-id file-id page-id position content] :as params}]
   (let [seqn    (retrieve-next-seqn conn file-id)
         now     (dt/now)
@@ -78,24 +82,6 @@
 
     (select-keys thread [:id :file-id :page-id])))
 
-(defn- create-comment-thread
-  [conn params]
-  (loop [sp (db/savepoint conn)
-         rc 0]
-    (let [res (ex/try (create-comment-thread* conn params))]
-      (cond
-        (and (instance? Throwable res)
-             (< rc 3))
-        (do
-          (db/rollback! conn sp)
-          (recur (db/savepoint conn)
-                 (inc rc)))
-
-        (instance? Throwable res)
-        (throw res)
-
-        :else res))))
-
 (defn- retrieve-page-name
   [conn {:keys [file-id page-id]}]
   (let [{:keys [data]} (db/get-by-id conn :file file-id)
@@ -188,7 +174,7 @@
                                  :content content})]
 
         ;; NOTE: this is done in SQL instead of using db/update!
-        ;; helper bacause currently the helper does not allow pass raw
+        ;; helper because currently the helper does not allow pass raw
         ;; function call parameters to the underlying prepared
         ;; statement; in a future when we fix/improve it, this can be
         ;; changed to use the helper.

backend/src/app/rpc/mutations/demo.clj

@@ -9,11 +9,10 @@
   (:require
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.rpc.mutations.profile :as profile]
-   [app.setup.initial-data :as sid]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [buddy.core.codecs :as bc]
@@ -34,20 +33,20 @@
         params   {:id id
                   :email email
                   :fullname fullname
-                  :is-demo true
-                  :deleted-at (dt/in-future cfg/deletion-delay)
+                  :is-active true
+                  :deleted-at (dt/in-future cf/deletion-delay)
                   :password password
-                  :props {:onboarding-viewed true}}]
+                  :props {}
+                  }]
 
-    (when-not (cfg/get :allow-demo-users)
+    (when-not (contains? cf/flags :demo-users)
       (ex/raise :type :validation
                 :code :demo-users-not-allowed
                 :hint "Demo users are disabled by config."))
 
     (db/with-atomic [conn pool]
       (->> (#'profile/create-profile conn params)
-           (#'profile/create-profile-relations conn)
-           (sid/load-initial-project! conn))
+           (#'profile/create-profile-relations conn))
 
       (with-meta {:email email
                   :password password}

backend/src/app/rpc/mutations/files.clj

@@ -11,11 +11,14 @@
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
+   [app.metrics :as mtx]
    [app.rpc.permissions :as perms]
    [app.rpc.queries.files :as files]
    [app.rpc.queries.projects :as proj]
    [app.storage.impl :as simpl]
+   [app.util.async :as async]
    [app.util.blob :as blob]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -25,6 +28,8 @@
 
 ;; --- Helpers & Specs
 
+(s/def ::frame-id ::us/uuid)
+(s/def ::file-id ::us/uuid)
 (s/def ::id ::us/uuid)
 (s/def ::name ::us/string)
 (s/def ::profile-id ::us/uuid)
@@ -180,7 +185,7 @@
                :library-file-id library-id}))
 
 
-;; --- Mutation: Update syncrhonization status of a link
+;; --- Mutation: Update synchronization status of a link
 
 (declare update-sync)
 
@@ -268,6 +273,7 @@
          (contains? o :changes-with-metadata)))))
 
 (sv/defmethod ::update-file
+  {::async/dispatch :blocking}
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
     (db/xact-lock! conn id)
@@ -279,11 +285,13 @@
 (defn- take-snapshot?
   "Defines the rule when file `data` snapshot should be saved."
   [{:keys [revn modified-at] :as file}]
-  ;; The snapshot will be saved every 20 changes or if the last
-  ;; modification is older than 3 hour.
-  (or (zero? (mod revn 20))
-      (> (inst-ms (dt/diff modified-at (dt/now)))
-         (inst-ms (dt/duration {:hours 3})))))
+  (let [freq    (or (cf/get :file-change-snapshot-every) 20)
+        timeout (or (cf/get :file-change-snapshot-timeout)
+                    (dt/duration {:hours 1}))]
+    (or (= 1 freq)
+        (zero? (mod revn freq))
+        (> (inst-ms (dt/diff modified-at (dt/now)))
+           (inst-ms timeout)))))
 
 (defn- delete-from-storage
   [{:keys [storage] :as cfg} file]
@@ -291,7 +299,7 @@
     (simpl/del-object backend file)))
 
 (defn- update-file
-  [{:keys [conn] :as cfg} {:keys [file changes changes-with-metadata session-id profile-id] :as params}]
+  [{:keys [conn metrics] :as cfg} {:keys [file changes changes-with-metadata session-id profile-id] :as params}]
   (when (> (:revn params)
            (:revn file))
 
@@ -305,10 +313,17 @@
                   (mapcat :changes changes-with-metadata)
                   changes)
 
+        changes (vec changes)
+
+        ;; Trace the number of changes processed
+        _       (mtx/run! metrics {:id :update-file-changes :inc (count changes)})
+
         ts      (dt/now)
         file    (-> (files/retrieve-data cfg file)
                     (update :revn inc)
                     (update :data (fn [data]
+                                    ;; Trace the length of bytes of processed data
+                                    (mtx/run! metrics {:id :update-file-bytes-processed :inc (alength data)})
                                     (-> data
                                         (blob/decode)
                                         (assoc :id (:id file))
@@ -400,8 +415,9 @@
   [conn project-id]
   (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]})))
 
-
-;; TEMPORARY FILE CREATION
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; TEMPORARY FILES (behaves differently)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::create-temp-file ::create-file)
 
@@ -411,6 +427,23 @@
     (proj/check-edition-permissions! conn profile-id project-id)
     (create-file conn (assoc params :deleted-at (dt/in-future {:days 1})))))
 
+(s/def ::update-temp-file
+  (s/keys :req-un [::changes ::revn ::session-id ::id]))
+
+(sv/defmethod ::update-temp-file
+  [{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}]
+  (db/with-atomic [conn pool]
+    (db/insert! conn :file-change
+                {:id (uuid/next)
+                 :session-id session-id
+                 :profile-id profile-id
+                 :created-at (dt/now)
+                 :file-id id
+                 :revn revn
+                 :data nil
+                 :changes (blob/encode changes)})
+    nil))
+
 (s/def ::persist-temp-file
   (s/keys :req-un [::id ::profile-id]))
 
@@ -418,6 +451,47 @@
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
     (files/check-edition-permissions! conn profile-id id)
-    (db/update! conn :file
-                {:deleted-at nil}
-                {:id id})))
+    (let [file (db/get-by-id conn :file id)
+          revs (db/query conn :file-change
+                         {:file-id id}
+                         {:order-by [[:revn :asc]]})
+          revn (count revs)]
+
+      (when (nil? (:deleted-at file))
+        (ex/raise :type :validation
+                  :code :cant-persist-already-persisted-file))
+
+      (loop [revs (seq revs)
+             data (blob/decode (:data file))]
+        (if-let [rev (first revs)]
+          (recur (rest revs)
+                 (->> rev :changes blob/decode (cp/process-changes data)))
+          (db/update! conn :file
+                      {:deleted-at nil
+                       :revn revn
+                       :data (blob/encode data)}
+                      {:id id})))
+
+      nil)))
+
+
+;; --- Mutation: Upsert frame thumbnail
+
+(def sql:upsert-frame-thumbnail
+  "insert into file_frame_thumbnail(file_id, frame_id, data)
+   values (?, ?, ?)
+       on conflict(file_id, frame_id) do
+          update set data = ?;")
+
+(s/def ::data ::us/string)
+(s/def ::upsert-frame-thumbnail
+  (s/keys :req-un [::profile-id ::file-id ::frame-id ::data]))
+
+(sv/defmethod ::upsert-frame-thumbnail
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id frame-id data]}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+    (db/exec-one! conn [sql:upsert-frame-thumbnail file-id frame-id data data])
+    nil))
+
+

backend/src/app/rpc/mutations/fonts.clj

@@ -9,14 +9,12 @@
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
-   [app.config :as cf]
    [app.db :as db]
    [app.media :as media]
    [app.rpc.queries.teams :as teams]
    [app.storage :as sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
-   [app.worker :as wrk]
    [clojure.spec.alpha :as s]))
 
 (declare create-font-variant)
@@ -40,51 +38,56 @@
 
 (sv/defmethod ::create-font-variant
   [{:keys [pool] :as cfg} {:keys [team-id profile-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [cfg (assoc cfg :conn conn)]
-      (teams/check-edition-permissions! conn profile-id team-id)
-      (create-font-variant cfg params))))
+  (teams/check-edition-permissions! pool profile-id team-id)
+  (create-font-variant cfg params))
 
 (defn create-font-variant
-  [{:keys [conn storage] :as cfg} {:keys [data] :as params}]
-  (let [data    (media/run cfg {:cmd :generate-fonts :input data :rlimit :font})
-        storage (media/configure-assets-storage storage conn)
+  [{:keys [storage pool] :as cfg} {:keys [data] :as params}]
+  (let [data    (media/run {:cmd :generate-fonts :input data})
+        storage (media/configure-assets-storage storage)]
 
-
-        otf     (when-let [fdata (get data "font/otf")]
-                  (sto/put-object storage {:content (sto/content fdata)
-                                           :content-type "font/otf"}))
-
-        ttf     (when-let [fdata (get data "font/ttf")]
-                  (sto/put-object storage {:content (sto/content fdata)
-                                           :content-type "font/ttf"}))
-
-        woff1   (when-let [fdata (get data "font/woff")]
-                  (sto/put-object storage {:content (sto/content fdata)
-                                           :content-type "font/woff"}))
-
-        woff2   (when-let [fdata (get data "font/woff2")]
-                  (sto/put-object storage {:content (sto/content fdata)
-                                           :content-type "font/woff2"}))]
-
-    (when (and (nil? otf)
-               (nil? ttf)
-               (nil? woff1)
-               (nil? woff2))
+    (when (and (not (contains? data "font/otf"))
+               (not (contains? data "font/ttf"))
+               (not (contains? data "font/woff"))
+               (not (contains? data "font/woff2")))
       (ex/raise :type :validation
                 :code :invalid-font-upload))
 
-    (db/insert! conn :team-font-variant
-                {:id (uuid/next)
-                 :team-id (:team-id params)
-                 :font-id (:font-id params)
-                 :font-family (:font-family params)
-                 :font-weight (:font-weight params)
-                 :font-style (:font-style params)
-                 :woff1-file-id (:id woff1)
-                 :woff2-file-id (:id woff2)
-                 :otf-file-id (:id otf)
-                 :ttf-file-id (:id ttf)})))
+    (let [otf   (when-let [fdata (get data "font/otf")]
+                  (sto/put-object storage {:content (sto/content fdata)
+                                           :content-type "font/otf"
+                                           :reference :team-font-variant
+                                           :touched-at (dt/now)}))
+
+          ttf   (when-let [fdata (get data "font/ttf")]
+                  (sto/put-object storage {:content (sto/content fdata)
+                                           :content-type "font/ttf"
+                                           :touched-at (dt/now)
+                                           :reference :team-font-variant}))
+
+          woff1 (when-let [fdata (get data "font/woff")]
+                  (sto/put-object storage {:content (sto/content fdata)
+                                           :content-type "font/woff"
+                                           :touched-at (dt/now)
+                                           :reference :team-font-variant}))
+
+          woff2 (when-let [fdata (get data "font/woff2")]
+                  (sto/put-object storage {:content (sto/content fdata)
+                                           :content-type "font/woff2"
+                                           :touched-at (dt/now)
+                                           :reference :team-font-variant}))]
+
+      (db/insert! pool :team-font-variant
+                  {:id (uuid/next)
+                   :team-id (:team-id params)
+                   :font-id (:font-id params)
+                   :font-family (:font-family params)
+                   :font-weight (:font-weight params)
+                   :font-style (:font-style params)
+                   :woff1-file-id (:id woff1)
+                   :woff2-file-id (:id woff2)
+                   :otf-file-id (:id otf)
+                   :ttf-file-id (:id ttf)}))))
 
 ;; --- UPDATE FONT FAMILY
 
@@ -129,13 +132,6 @@
   (db/with-atomic [conn pool]
     (teams/check-edition-permissions! conn profile-id team-id)
 
-    ;; Schedule object deletion
-    (wrk/submit! {::wrk/task :delete-object
-                  ::wrk/delay cf/deletion-delay
-                  ::wrk/conn conn
-                  :id id
-                  :type :team-font-variant})
-
     (db/update! conn :team-font-variant
                 {:deleted-at (dt/now)}
                 {:id id :team-id team-id})

backend/src/app/rpc/mutations/ldap.clj

@@ -7,13 +7,13 @@
 (ns app.rpc.mutations.ldap
   (:require
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cfg]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.rpc.mutations.profile :as profile-m]
    [app.rpc.queries.profile :as profile-q]
-   [app.util.logging :as l]
    [app.util.services :as sv]
    [clj-ldap.client :as ldap]
    [clojure.spec.alpha :as s]
@@ -56,7 +56,7 @@
   (s/keys :req-un [::email ::password]
           :opt-un [::invitation-token]))
 
-(sv/defmethod ::login-with-ldap {:auth false :rlimit :password}
+(sv/defmethod ::login-with-ldap {:auth false}
   [{:keys [pool session tokens] :as cfg} params]
   (db/with-atomic [conn pool]
     (let [info (authenticate params)

backend/src/app/rpc/mutations/management.clj

@@ -39,18 +39,30 @@
   [file index]
   (letfn [(process-form [form]
             (cond-> form
-              ;; Relink Components
+              ;; Relink library items
               (and (map? form)
                    (uuid? (:component-file form)))
               (update :component-file #(get index % %))
 
+              (and (map? form)
+                   (uuid? (:fill-color-ref-file form)))
+              (update :fill-color-ref-file #(get index % %))
+
+              (and (map? form)
+                   (uuid? (:stroke-color-ref-file form)))
+              (update :stroke-color-ref-file #(get index % %))
+
+              (and (map? form)
+                   (uuid? (:typography-ref-file form)))
+              (update :typography-ref-file #(get index % %))
+
               ;; Relink Image Shapes
               (and (map? form)
                    (map? (:metadata form))
                    (= :image (:type form)))
               (update-in [:metadata :id] #(get index % %))))
 
-          ;; A function responsible to analize all file data and
+          ;; A function responsible to analyze all file data and
           ;; replace the old :component-file reference with the new
           ;; ones, using the provided file-index
           (relink-shapes [data]
@@ -282,7 +294,7 @@
       ;; move all files to the project
       (db/exec-one! conn [sql:move-files project-id fids])
 
-      ;; delete posible broken relations on moved files
+      ;; delete possible broken relations on moved files
       (db/exec-one! conn [sql:delete-broken-relations pids])
 
       nil)))
@@ -317,7 +329,7 @@
                   {:team-id team-id}
                   {:id project-id})
 
-      ;; delete posible broken relations on moved files
+      ;; delete possible broken relations on moved files
       (db/exec-one! conn [sql:delete-broken-relations pids])
 
       nil)))

backend/src/app/rpc/mutations/media.clj

@@ -10,10 +10,13 @@
    [app.common.media :as cm]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.media :as media]
    [app.rpc.queries.teams :as teams]
+   [app.rpc.rlimit :as rlimit]
    [app.storage :as sto]
+   [app.util.async :as async]
    [app.util.http :as http]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -47,12 +50,12 @@
           :opt-un [::id]))
 
 (sv/defmethod ::upload-file-media-object
+  {::rlimit/permits (cf/get :rlimit-image)
+   ::async/dispatch :default}
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
-  (db/with-atomic [conn pool]
-    (let [file (select-file conn file-id)]
-      (teams/check-edition-permissions! conn profile-id (:team-id file))
-      (-> (assoc cfg :conn conn)
-          (create-file-media-object params)))))
+  (let [file (select-file pool file-id)]
+    (teams/check-edition-permissions! pool profile-id (:team-id file))
+    (create-file-media-object cfg params)))
 
 (defn- big-enough-for-thumbnail?
   "Checks if the provided image info is big enough for
@@ -74,6 +77,9 @@
                 :code :unable-to-access-to-url
                 :cause e))))
 
+;; TODO: we need to check the size before fetch resource, if not we
+;; can start downloading very big object and cause OOM errors.
+
 (defn- download-media
   [{:keys [storage] :as cfg} url]
   (let [result (fetch-url url)
@@ -87,45 +93,76 @@
     (-> (assoc storage :backend :tmp)
         (sto/put-object {:content (sto/content data)
                          :content-type mtype
+                         :reference :file-media-object
                          :expired-at (dt/in-future {:minutes 30})}))))
 
+;; NOTE: we use the `on conflict do update` instead of `do nothing`
+;; because postgresql does not returns anything if no update is
+;; performed, the `do update` does the trick.
+
+(def sql:create-file-media-object
+  "insert into file_media_object (id, file_id, is_local, name, media_id, thumbnail_id, width, height, mtype)
+   values (?, ?, ?, ?, ?, ?, ?, ?, ?)
+       on conflict (id) do update set created_at=file_media_object.created_at
+       returning *")
+
+;; NOTE: the following function executes without a transaction, this
+;; means that if something fails in the middle of this function, it
+;; will probably leave leaked/unreferenced objects in the database and
+;; probably in the storage layer. For handle possible object leakage,
+;; we create all media objects marked as touched, this ensures that if
+;; something fails, all leaked (already created storage objects) will
+;; be eventually marked as deleted by the touched-gc task.
+;;
+;; The touched-gc task, performs periodic analisis of all touched
+;; storage objects and check references of it. This is the reason why
+;; `reference` metadata exists: it indicates the name of the table
+;; witch holds the reference to storage object (it some kind of
+;; inverse, soft referential integrity).
 
 (defn create-file-media-object
-  [{:keys [conn storage] :as cfg} {:keys [id file-id is-local name content] :as params}]
+  [{:keys [storage pool] :as cfg} {:keys [id file-id is-local name content] :as params}]
   (media/validate-media-type (:content-type content))
-  (let [storage      (media/configure-assets-storage storage conn)
-        source-path  (fs/path (:tempfile content))
+  (let [source-path  (fs/path (:tempfile content))
         source-mtype (:content-type content)
-        source-info  (media/run cfg {:cmd :info :input {:path source-path :mtype source-mtype}})
+        source-info  (media/run {:cmd :info :input {:path source-path :mtype source-mtype}})
+        storage      (media/configure-assets-storage storage)
 
         thumb        (when (and (not (svg-image? source-info))
                                 (big-enough-for-thumbnail? source-info))
-                       (media/run cfg (assoc thumbnail-options
-                                             :cmd :generic-thumbnail
-                                             :input {:mtype (:mtype source-info)
-                                                     :path source-path})))
+                       (media/run (assoc thumbnail-options
+                                         :cmd :generic-thumbnail
+                                         :input {:mtype (:mtype source-info)
+                                                 :path source-path})))
 
         image        (if (= (:mtype source-info) "image/svg+xml")
                        (let [data (slurp source-path)]
-                         (sto/put-object storage {:content (sto/content data)
-                                                  :content-type (:mtype source-info)}))
-                       (sto/put-object storage {:content (sto/content source-path)
-                                                :content-type (:mtype source-info)}))
+                         (sto/put-object storage
+                                         {:content (sto/content data)
+                                          :content-type (:mtype source-info)
+                                          :reference :file-media-object
+                                          :touched-at (dt/now)}))
+                       (sto/put-object storage
+                                       {:content (sto/content source-path)
+                                        :content-type (:mtype source-info)
+                                        :reference :file-media-object
+                                        :touched-at (dt/now)}))
 
         thumb        (when thumb
-                       (sto/put-object storage {:content (sto/content (:data thumb) (:size thumb))
-                                                :content-type (:mtype thumb)}))]
-    (db/insert! conn :file-media-object
-                {:id (or id (uuid/next))
-                 :file-id file-id
-                 :is-local is-local
-                 :name name
-                 :media-id (:id image)
-                 :thumbnail-id (:id thumb)
-                 :width  (:width source-info)
-                 :height (:height source-info)
-                 :mtype  source-mtype})))
+                       (sto/put-object storage
+                                       {:content (sto/content (:data thumb) (:size thumb))
+                                        :content-type (:mtype thumb)
+                                        :reference :file-media-object
+                                        :touched-at (dt/now)}))]
 
+    (db/exec-one! pool [sql:create-file-media-object
+                        (or id (uuid/next))
+                        file-id is-local name
+                        (:id image)
+                        (:id thumb)
+                        (:width source-info)
+                        (:height source-info)
+                        source-mtype])))
 
 ;; --- Create File Media Object (from URL)
 
@@ -134,20 +171,19 @@
           :opt-un [::id ::name]))
 
 (sv/defmethod ::create-file-media-object-from-url
+  {::rlimit/permits (cf/get :rlimit-image)
+   ::async/dispatch :default}
   [{:keys [pool storage] :as cfg} {:keys [profile-id file-id url name] :as params}]
-  (db/with-atomic [conn pool]
-    (let [file (select-file conn file-id)]
-      (teams/check-edition-permissions! conn profile-id (:team-id file))
-      (let [mobj    (download-media cfg url)
-            content {:filename "tempfile"
-                     :size (:size mobj)
-                     :tempfile (sto/get-object-path storage mobj)
-                     :content-type (:content-type (meta mobj))}
-            params' (merge params {:content content
-                                   :name (or name (:filename content))})]
-        (-> (assoc cfg :conn conn)
-            (create-file-media-object params'))))))
+  (let [file (select-file pool file-id)]
+    (teams/check-edition-permissions! pool profile-id (:team-id file))
+    (let [mobj    (download-media cfg url)
+          content {:filename "tempfile"
+                   :size (:size mobj)
+                   :tempfile (sto/get-object-path storage mobj)
+                   :content-type (:content-type (meta mobj))}]
 
+      (->> (merge params {:content content :name (or name (:filename content))})
+           (create-file-media-object cfg)))))
 
 ;; --- Clone File Media object (Upload and create from url)
 
@@ -179,7 +215,6 @@
                  :height (:height mobj)
                  :mtype  (:mtype mobj)})))
 
-
 ;; --- HELPERS
 
 (def ^:private

backend/src/app/rpc/mutations/profile.clj

@@ -12,14 +12,14 @@
    [app.config :as cf]
    [app.db :as db]
    [app.emails :as eml]
-   [app.http.oauth :refer [extract-props]]
+   [app.http.oauth :refer [extract-utm-props]]
    [app.loggers.audit :as audit]
    [app.media :as media]
-   [app.rpc.mutations.projects :as projects]
    [app.rpc.mutations.teams :as teams]
    [app.rpc.queries.profile :as profile]
-   [app.setup.initial-data :as sid]
+   [app.rpc.rlimit :as rlimit]
    [app.storage :as sto]
+   [app.util.async :as async]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [buddy.hashers :as hashers]
@@ -38,7 +38,6 @@
 (s/def ::theme ::us/string)
 (s/def ::invitation-token ::us/not-empty-string)
 
-(declare annotate-profile-register)
 (declare check-profile-existence!)
 (declare create-profile)
 (declare create-profile-relations)
@@ -99,7 +98,7 @@
 
 (sv/defmethod ::prepare-register-profile {:auth false}
   [{:keys [pool tokens] :as cfg} params]
-  (when-not (cf/get :registration-enabled)
+  (when-not (contains? cf/flags :registration)
     (ex/raise :type :restriction
               :code :registration-disabled))
 
@@ -109,7 +108,7 @@
                 :code :email-domain-is-not-allowed)))
 
   ;; Don't allow proceed in preparing registration if the profile is
-  ;; already reported as spamer.
+  ;; already reported as spammer.
   (when (eml/has-bounce-reports? pool (:email params))
     (ex/raise :type :validation
               :code :email-has-permanent-bounces
@@ -117,78 +116,77 @@
 
   (check-profile-existence! pool params)
 
-  (let [params (assoc params
-                      :backend "penpot"
-                      :iss :prepared-register
-                      :exp (dt/in-future "48h"))
+  (when (= (str/lower (:email params))
+           (str/lower (:password params)))
+    (ex/raise :type :validation
+              :code :email-as-password
+              :hint "you can't use your email as password"))
+
+  (let [params {:email (:email params)
+                :invitation-token (:invitation-token params)
+                :backend "penpot"
+                :iss :prepared-register
+                :exp (dt/in-future "48h")}
         token  (tokens :generate params)]
     {:token token}))
 
 ;; --- MUTATION: Register Profile
 
-(s/def ::accept-terms-and-privacy ::us/boolean)
-(s/def ::accept-newsletter-subscription ::us/boolean)
 (s/def ::token ::us/not-empty-string)
-
 (s/def ::register-profile
-  (s/keys :req-un [::token ::fullname
-                   ::accept-terms-and-privacy]
-          :opt-un [::accept-newsletter-subscription]))
+  (s/keys :req-un [::token ::fullname]))
 
-(sv/defmethod ::register-profile {:auth false :rlimit :password}
+(sv/defmethod ::register-profile
+  {:auth false ::rlimit/permits (cf/get :rlimit-password)}
   [{:keys [pool] :as cfg} params]
-  (when-not (:accept-terms-and-privacy params)
-    (ex/raise :type :validation
-              :code :invalid-terms-and-privacy))
-
   (db/with-atomic [conn pool]
-    (let [cfg     (assoc cfg :conn conn)]
-      (register-profile cfg params))))
-
-(defn- annotate-profile-register
-  "A helper for properly increase the profile-register metric once the
-  transaction is completed."
-  [metrics]
-  (fn []
-    ((get-in metrics [:definitions :profile-register]) :inc)))
+    (-> (assoc cfg :conn conn)
+        (register-profile params))))
 
 (defn register-profile
-  [{:keys [conn tokens session metrics] :as cfg} {:keys [token] :as params}]
-  (let [claims (tokens :verify {:token token :iss :prepared-register})
-        params (merge params claims)]
+  [{:keys [conn tokens session] :as cfg} {:keys [token] :as params}]
+  (let [claims    (tokens :verify {:token token :iss :prepared-register})
+        params    (merge params claims)]
+
     (check-profile-existence! conn params)
-    (let [profile (->> params
-                       (create-profile conn)
-                       (create-profile-relations conn)
-                       (decode-profile-row))]
-      (sid/load-initial-project! conn profile)
+
+    (let [is-active  (or (:is-active params)
+                         (contains? cf/flags :insecure-register))
+          profile    (->> (assoc params :is-active is-active)
+                          (create-profile conn)
+                          (create-profile-relations conn)
+                          (decode-profile-row))
+
+          invitation (when-let [token (:invitation-token params)]
+                       (tokens :verify {:token token :iss :team-invitation}))]
 
       (cond
-        ;; If invitation token comes in params, this is because the
-        ;; user comes from team-invitation process; in this case,
-        ;; regenerate token and send back to the user a new invitation
-        ;; token (and mark current session as logged).
-        (some? (:invitation-token params))
-        (let [token (:invitation-token params)
-              claims (tokens :verify {:token token :iss :team-invitation})
-              claims (assoc claims
-                            :member-id  (:id profile)
-                            :member-email (:email profile))
+        ;; If invitation token comes in params, this is because the user comes from team-invitation process;
+        ;; in this case, regenerate token and send back to the user a new invitation token (and mark current
+        ;; session as logged). This happens only if the invitation email matches with the register email.
+        (and (some? invitation) (= (:email profile) (:member-email invitation)))
+        (let [claims (assoc invitation :member-id  (:id profile))
               token  (tokens :generate claims)
               resp   {:invitation-token token}]
           (with-meta resp
             {:transform-response ((:create session) (:id profile))
-             :before-complete (annotate-profile-register metrics)
              ::audit/props (audit/profile->props profile)
              ::audit/profile-id (:id profile)}))
 
         ;; If auth backend is different from "penpot" means user is
-        ;; registring using third party auth mechanism; in this case
+        ;; registering using third party auth mechanism; in this case
         ;; we need to mark this session as logged.
         (not= "penpot" (:auth-backend profile))
         (with-meta (profile/strip-private-attrs profile)
           {:transform-response ((:create session) (:id profile))
-           :before-complete (annotate-profile-register metrics)
+           ::audit/props (audit/profile->props profile)
+           ::audit/profile-id (:id profile)})
+
+        ;; If the `:enable-insecure-register` flag is set, we proceed
+        ;; to sign in the user directly, without email verification.
+        (true? is-active)
+        (with-meta (profile/strip-private-attrs profile)
+          {:transform-response ((:create session) (:id profile))
            ::audit/props (audit/profile->props profile)
            ::audit/profile-id (:id profile)})
 
@@ -202,7 +200,6 @@
               ptoken (tokens :generate-predefined
                              {:iss :profile-identity
                               :profile-id (:id profile)})]
-
           (eml/send! {::eml/conn conn
                       ::eml/factory eml/register
                       :public-uri (:public-uri cfg)
@@ -212,8 +209,7 @@
                       :extra-data ptoken})
 
           (with-meta profile
-            {:before-complete (annotate-profile-register metrics)
-             ::audit/props (audit/profile->props profile)
+            {::audit/props (audit/profile->props profile)
              ::audit/profile-id (:id profile)}))))))
 
 (defn create-profile
@@ -222,23 +218,22 @@
   [conn params]
   (let [id        (or (:id params) (uuid/next))
 
-        props     (-> (extract-props params)
+        props     (-> (extract-utm-props params)
                       (merge (:props params))
-                      (assoc :accept-terms-and-privacy (:accept-terms-and-privacy params true))
-                      (assoc :accept-newsletter-subscription (:accept-newsletter-subscription params false))
                       (db/tjson))
 
         password  (if-let [password (:password params)]
                     (derive-password password)
                     "!")
 
-        locale    (as-> (:locale params) locale
-                    (and (string? locale) (not (str/blank? locale)) locale))
+        locale    (:locale params)
+        locale    (when (and (string? locale) (not (str/blank? locale)))
+                    locale)
 
         backend   (:backend params "penpot")
         is-demo   (:is-demo params false)
         is-muted  (:is-muted params false)
-        is-active (:is-active params (or (not= "penpot" backend) is-demo))
+        is-active (:is-active params false)
         email     (str/lower (:email params))
 
         params    {:id id
@@ -263,28 +258,15 @@
                       :code :email-already-exists
                       :cause e)))))))
 
-
 (defn create-profile-relations
   [conn profile]
-  (let [team    (teams/create-team conn {:profile-id (:id profile)
-                                         :name "Default"
-                                         :is-default true})
-        project (projects/create-project conn {:profile-id (:id profile)
-                                               :team-id (:id team)
-                                               :name "Drafts"
-                                               :is-default true})
-        params  {:team-id (:id team)
-                 :profile-id (:id profile)
-                 :project-id (:id project)
-                 :role :owner}]
-
-    (teams/create-team-role conn params)
-    (projects/create-project-role conn params)
-
+  (let [team (teams/create-team conn {:profile-id (:id profile)
+                                      :name "Default"
+                                      :is-default true})]
     (-> profile
         (profile/strip-private-attrs)
         (assoc :default-team-id (:id team))
-        (assoc :default-project-id (:id project)))))
+        (assoc :default-project-id (:default-project-id team)))))
 
 ;; --- MUTATION: Login
 
@@ -295,7 +277,10 @@
   (s/keys :req-un [::email ::password]
           :opt-un [::scope ::invitation-token]))
 
-(sv/defmethod ::login {:auth false :rlimit :password}
+(sv/defmethod ::login
+  {:auth false
+   ::async/dispatch :default
+   ::rlimit/permits (cf/get :rlimit-password)}
   [{:keys [pool session tokens] :as cfg} {:keys [email password] :as params}]
   (letfn [(check-password [profile password]
             (when (= (:password profile) "!")
@@ -316,39 +301,33 @@
             profile)]
 
     (db/with-atomic [conn pool]
-      (let [profile (->> (profile/retrieve-profile-data-by-email conn email)
-                         (validate-profile)
-                         (profile/strip-private-attrs)
-                         (profile/populate-additional-data conn)
-                         (decode-profile-row))]
-        (if-let [token (:invitation-token params)]
-          ;; If the request comes with an invitation token, this means
-          ;; that user wants to accept it with different user. A very
-          ;; strange case but still can happen. In this case, we
-          ;; proceed in the same way as in register: regenerate the
-          ;; invitation token and return it to the user for proper
-          ;; invitation acceptation.
-          (let [claims (tokens :verify {:token token :iss :team-invitation})
-                claims (assoc claims
-                              :member-id  (:id profile)
-                              :member-email (:email profile))
-                token  (tokens :generate claims)]
-            (with-meta {:invitation-token token}
-              {:transform-response ((:create session) (:id profile))
-               ::audit/props (audit/profile->props profile)
-               ::audit/profile-id (:id profile)}))
+      (let [profile    (->> (profile/retrieve-profile-data-by-email conn email)
+                            (validate-profile)
+                            (profile/strip-private-attrs)
+                            (profile/populate-additional-data conn)
+                            (decode-profile-row))
 
-          (with-meta profile
-            {:transform-response ((:create session) (:id profile))
-             ::audit/props (audit/profile->props profile)
-             ::audit/profile-id (:id profile)}))))))
+            invitation (when-let [token (:invitation-token params)]
+                         (tokens :verify {:token token :iss :team-invitation}))
+
+            ;; If invitation member-id does not matches the profile-id, we just proceed to ignore the
+            ;; invitation because invitations matches exactly; and user can't loging with other email and
+            ;; accept invitation with other email
+            response   (if (and (some? invitation) (= (:id profile) (:member-id invitation)))
+                         {:invitation-token (:invitation-token params)}
+                         profile)]
+
+        (with-meta response
+          {:transform-response ((:create session) (:id profile))
+           ::audit/props (audit/profile->props profile)
+           ::audit/profile-id (:id profile)})))))
 
 ;; --- MUTATION: Logout
 
 (s/def ::logout
-  (s/keys :req-un [::profile-id]))
+  (s/keys :opt-un [::profile-id]))
 
-(sv/defmethod ::logout
+(sv/defmethod ::logout {:auth false}
   [{:keys [session] :as cfg} _]
   (with-meta {}
     {:transform-response (:delete session)}))
@@ -357,11 +336,14 @@
 
 (defn- update-profile
   [conn {:keys [id fullname lang theme] :as params}]
-  (db/update! conn :profile
-              {:fullname fullname
-               :lang lang
-               :theme theme}
-              {:id id}))
+  (let [profile (db/update! conn :profile
+                            {:fullname fullname
+                             :lang lang
+                             :theme theme}
+                            {:id id})]
+    (-> profile
+        (profile/decode-profile-row)
+        (profile/strip-private-attrs))))
 
 (s/def ::update-profile
   (s/keys :req-un [::id ::fullname]
@@ -370,24 +352,40 @@
 (sv/defmethod ::update-profile
   [{:keys [pool] :as cfg} params]
   (db/with-atomic [conn pool]
-    (update-profile conn params)
-    nil))
+    (let [profile (update-profile conn params)]
+      (with-meta profile
+        {::audit/props (audit/profile->props profile)}))))
 
 ;; --- MUTATION: Update Password
 
 (declare validate-password!)
 (declare update-profile-password!)
+(declare invalidate-profile-session!)
 
 (s/def ::update-profile-password
   (s/keys :req-un [::profile-id ::password ::old-password]))
 
-(sv/defmethod ::update-profile-password {:rlimit :password}
+(sv/defmethod ::update-profile-password
+  {::rlimit/permits (cf/get :rlimit-password)}
   [{:keys [pool] :as cfg} {:keys [password] :as params}]
   (db/with-atomic [conn pool]
-    (let [profile (validate-password! conn params)]
+    (let [profile    (validate-password! conn params)
+          session-id (:app.rpc/session-id params)]
+      (when (= (str/lower (:email profile))
+               (str/lower (:password params)))
+        (ex/raise :type :validation
+                  :code :email-as-password
+                  :hint "you can't use your email as password"))
       (update-profile-password! conn (assoc profile :password password))
+      (invalidate-profile-session! conn (:id profile) session-id)
       nil)))
 
+(defn- invalidate-profile-session!
+  "Removes all sessions except the current one."
+  [conn profile-id session-id]
+  (let [sql "delete from http_session where profile_id = ? and id != ?"]
+    (:next.jdbc/update-count (db/exec-one! conn [sql profile-id session-id]))))
+
 (defn- validate-password!
   [conn {:keys [profile-id old-password] :as params}]
   (let [profile (db/get-by-id conn :profile profile-id)]
@@ -402,7 +400,6 @@
               {:password (derive-password password)}
               {:id id}))
 
-
 ;; --- MUTATION: Update Photo
 
 (declare update-profile-photo)
@@ -414,11 +411,12 @@
   (s/keys :req-un [::profile-id ::file]))
 
 (sv/defmethod ::update-profile-photo
+  {::rlimit/permits (cf/get :rlimit-image)}
   [{:keys [pool storage] :as cfg} {:keys [profile-id file] :as params}]
   (db/with-atomic [conn pool]
     (media/validate-media-type (:content-type file) #{"image/jpeg" "image/png" "image/webp"})
-    (media/run cfg {:cmd :info :input {:path (:tempfile file)
-                                       :mtype (:content-type file)}})
+    (media/run {:cmd :info :input {:path (:tempfile file)
+                                   :mtype (:content-type file)}})
 
     (let [profile (db/get-by-id conn :profile profile-id)
           storage (media/configure-assets-storage storage conn)
@@ -443,7 +441,7 @@
 ;; --- MUTATION: Request Email Change
 
 (declare request-email-change)
-(declare change-email-inmediatelly)
+(declare change-email-immediately)
 
 (s/def ::request-email-change
   (s/keys :req-un [::email]))
@@ -456,11 +454,12 @@
           params  (assoc params
                          :profile profile
                          :email (str/lower email))]
-      (if (cf/get :smtp-enabled)
+      (if (or (cf/get :smtp-enabled)
+              (contains? cf/flags :smtp))
         (request-email-change cfg params)
-        (change-email-inmediatelly cfg params)))))
+        (change-email-immediately cfg params)))))
 
-(defn- change-email-inmediatelly
+(defn- change-email-immediately
   [{:keys [conn]} {:keys [profile email] :as params}]
   (when (not= email (:email profile))
     (check-profile-existence! conn params))
@@ -563,7 +562,8 @@
 (s/def ::recover-profile
   (s/keys :req-un [::token ::password]))
 
-(sv/defmethod ::recover-profile {:auth false :rlimit :password}
+(sv/defmethod ::recover-profile
+  {:auth false ::rlimit/permits (cf/get :rlimit-password)}
   [{:keys [pool tokens] :as cfg} {:keys [token password]}]
   (letfn [(validate-token [token]
             (let [tdata (tokens :verify {:token token :iss :password-recovery})]
@@ -589,15 +589,20 @@
   (db/with-atomic [conn pool]
     (let [profile (profile/retrieve-profile-data conn profile-id)
           props   (reduce-kv (fn [props k v]
-                               (if (nil? v)
-                                 (dissoc props k)
-                                 (assoc props k v)))
+                               ;; We don't accept namespaced keys
+                               (if (simple-ident? k)
+                                 (if (nil? v)
+                                   (dissoc props k)
+                                   (assoc props k v))
+                                 props))
                              (:props profile)
                              props)]
+
       (db/update! conn :profile
                   {:props (db/tjson props)}
                   {:id profile-id})
-      nil)))
+
+      (profile/filter-profile-props props))))
 
 
 ;; --- MUTATION: Delete Profile
@@ -638,7 +643,7 @@
   (let [rows (db/exec! conn [sql:owned-teams profile-id])]
     ;; If we found owned teams with more than one profile we don't
     ;; allow delete profile until the user properly transfer ownership
-    ;; or explictly removes all participants from the team.
+    ;; or explicitly removes all participants from the team.
     (when (some #(> (:num-profiles %) 1) rows)
       (ex/raise :type :validation
                 :code :owner-teams-with-people

backend/src/app/rpc/mutations/share_link.clj

@@ -0,0 +1,72 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.mutations.share-link
+  "Share link related rpc mutation methods."
+  (:require
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.rpc.queries.files :as files]
+   [app.util.services :as sv]
+   [clojure.spec.alpha :as s]))
+
+;; --- Helpers & Specs
+
+(s/def ::id ::us/uuid)
+(s/def ::profile-id ::us/uuid)
+(s/def ::file-id ::us/uuid)
+(s/def ::flags (s/every ::us/string :kind set?))
+(s/def ::pages (s/every ::us/uuid :kind set?))
+
+;; --- Mutation: Create Share Link
+
+(declare create-share-link)
+
+(s/def ::create-share-link
+  (s/keys :req-un [::profile-id ::file-id ::flags]
+          :opt-un [::pages]))
+
+(sv/defmethod ::create-share-link
+  "Creates a share-link object.
+
+  Share links are resources that allows external users access to
+  specific files with specific permissions (flags)."
+
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-edition-permissions! conn profile-id file-id)
+    (create-share-link conn params)))
+
+(defn create-share-link
+  [conn {:keys [profile-id file-id pages flags]}]
+  (let [pages (db/create-array conn "uuid" pages)
+        flags (->> (map name flags)
+                   (db/create-array conn "text"))
+        slink (db/insert! conn :share-link
+                          {:id (uuid/next)
+                           :file-id file-id
+                           :flags flags
+                           :pages pages
+                           :owner-id profile-id})]
+    (-> slink
+        (update :pages db/decode-pgarray #{})
+        (update :flags db/decode-pgarray #{}))))
+
+;; --- Mutation: Delete Share Link
+
+(declare delete-share-link)
+
+(s/def ::delete-share-link
+  (s/keys :req-un [::profile-id ::id]))
+
+(sv/defmethod ::delete-share-link
+  [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [slink (db/get-by-id conn :share-link id)]
+      (files/check-edition-permissions! conn profile-id (:file-id slink))
+      (db/delete! conn :share-link {:id id})
+      nil)))

backend/src/app/rpc/mutations/teams.clj

@@ -10,6 +10,7 @@
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.db :as db]
    [app.emails :as eml]
    [app.media :as media]
@@ -17,6 +18,7 @@
    [app.rpc.permissions :as perms]
    [app.rpc.queries.profile :as profile]
    [app.rpc.queries.teams :as teams]
+   [app.rpc.rlimit :as rlimit]
    [app.storage :as sto]
    [app.util.services :as sv]
    [app.util.time :as dt]
@@ -32,6 +34,7 @@
 ;; --- Mutation: Create Team
 
 (declare create-team)
+(declare create-team-entry)
 (declare create-team-role)
 (declare create-team-default-project)
 
@@ -42,15 +45,21 @@
 (sv/defmethod ::create-team
   [{:keys [pool] :as cfg} params]
   (db/with-atomic [conn pool]
-    (let [team   (create-team conn params)
-          params (assoc params
-                        :team-id (:id team)
-                        :role :owner)]
-      (create-team-role conn params)
-      (create-team-default-project conn params)
-      team)))
+    (create-team conn params)))
 
 (defn create-team
+  "This is a complete team creation process, it creates the team
+  object and all related objects (default role and default project)."
+  [conn params]
+  (let [team    (create-team-entry conn params)
+        params  (assoc params
+                       :team-id (:id team)
+                       :role :owner)
+        project (create-team-default-project conn params)]
+    (create-team-role conn params)
+    (assoc team :default-project-id (:id project))))
+
+(defn- create-team-entry
   [conn {:keys [id name is-default] :as params}]
   (let [id         (or id (uuid/next))
         is-default (if (boolean? is-default) is-default false)]
@@ -59,23 +68,24 @@
                  :name name
                  :is-default is-default})))
 
-(defn create-team-role
+(defn- create-team-role
   [conn {:keys [team-id profile-id role] :as params}]
   (let [params {:team-id team-id
                 :profile-id profile-id}]
     (->> (perms/assign-role-flags params role)
          (db/insert! conn :team-profile-rel))))
 
-(defn create-team-default-project
+(defn- create-team-default-project
   [conn {:keys [team-id profile-id] :as params}]
   (let [project {:id (uuid/next)
                  :team-id team-id
                  :name "Drafts"
-                 :is-default true}]
-    (projects/create-project conn project)
+                 :is-default true}
+        project (projects/create-project conn project)]
     (projects/create-project-role conn {:project-id (:id project)
                                         :profile-id profile-id
-                                        :role :owner})))
+                                        :role :owner})
+    project))
 
 ;; --- Mutation: Update Team
 
@@ -94,24 +104,53 @@
 
 ;; --- Mutation: Leave Team
 
+(declare role->params)
+
+(s/def ::reassign-to ::us/uuid)
 (s/def ::leave-team
-  (s/keys :req-un [::profile-id ::id]))
+  (s/keys :req-un [::profile-id ::id]
+          :opt-un [::reassign-to]))
 
 (sv/defmethod ::leave-team
-  [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
+  [{:keys [pool] :as cfg} {:keys [id profile-id reassign-to]}]
   (db/with-atomic [conn pool]
-    (let [perms   (teams/check-read-permissions! conn profile-id id)
+    (let [perms   (teams/get-permissions conn profile-id id)
           members (teams/retrieve-team-members conn id)]
 
-      (when (some :is-owner perms)
+      (cond
+        ;; we can only proceed if there are more members in the team
+        ;; besides the current profile
+        (<= (count members) 1)
+        (ex/raise :type :validation
+                  :code :no-enough-members-for-leave
+                  :context {:members (count members)})
+
+        ;; if the `reassign-to` is filled and has a different value
+        ;; than the current profile-id, we proceed to reassing the
+        ;; owner role to profile identified by the `reassign-to`.
+        (and reassign-to (not= reassign-to profile-id))
+        (let [member (d/seek #(= reassign-to (:id %)) members)]
+          (when-not member
+            (ex/raise :type :not-found :code :member-does-not-exist))
+
+          ;; unasign owner role to current profile
+          (db/update! conn :team-profile-rel
+                      {:is-owner false}
+                      {:team-id id
+                       :profile-id profile-id})
+
+          ;; assign owner role to new profile
+          (db/update! conn :team-profile-rel
+                      (role->params :owner)
+                      {:team-id id :profile-id reassign-to}))
+
+        ;; and finally, if all other conditions does not match and the
+        ;; current profile is owner, we dont allow it because there
+        ;; must always be an owner.
+        (:is-owner perms)
         (ex/raise :type :validation
                   :code :owner-cant-leave-team
-                  :hint "reasing owner before leave"))
-
-      (when-not (> (count members) 1)
-        (ex/raise :type :validation
-                  :code :cant-leave-team
-                  :context {:members (count members)}))
+                  :hint "releasing owner before leave"))
 
       (db/delete! conn :team-profile-rel
                   {:profile-id profile-id
@@ -119,21 +158,20 @@
 
       nil)))
 
-
 ;; --- Mutation: Delete Team
 
 (s/def ::delete-team
   (s/keys :req-un [::profile-id ::id]))
 
 ;; TODO: right now just don't allow delete default team, in future it
-;; should raise a speific exception for signal that this acction is
+;; should raise a specific exception for signal that this action is
 ;; not allowed.
 
 (sv/defmethod ::delete-team
   [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}]
   (db/with-atomic [conn pool]
-    (let [perms (teams/check-edition-permissions! conn profile-id id)]
-      (when-not (some :is-owner perms)
+    (let [perms (teams/get-permissions conn profile-id id)]
+      (when-not (:is-owner perms)
         (ex/raise :type :validation
                   :code :only-owner-can-delete-team))
 
@@ -146,7 +184,6 @@
 ;; --- Mutation: Team Update Role
 
 (declare retrieve-team-member)
-(declare role->params)
 
 (s/def ::team-id ::us/uuid)
 (s/def ::member-id ::us/uuid)
@@ -161,17 +198,16 @@
 (sv/defmethod ::update-team-member-role
   [{:keys [pool] :as cfg} {:keys [team-id profile-id member-id role] :as params}]
   (db/with-atomic [conn pool]
-    (let [perms   (teams/check-read-permissions! conn profile-id team-id)
-
+    (let [perms (teams/get-permissions conn profile-id team-id)
           ;; We retrieve all team members instead of query the
           ;; database for a single member. This is just for
-          ;; convenience, if this bocomes a bottleneck or problematic,
-          ;; we will change it to more efficient fetch mechanims.
+          ;; convenience, if this becomes a bottleneck or problematic,
+          ;; we will change it to more efficient fetch mechanisms.
           members (teams/retrieve-team-members conn team-id)
           member  (d/seek #(= member-id (:id %)) members)
 
-          is-owner? (some :is-owner perms)
-          is-admin? (some :is-admin perms)]
+          is-owner? (:is-owner perms)
+          is-admin? (:is-admin perms)]
 
       ;; If no member is found, just 404
       (when-not member
@@ -224,9 +260,9 @@
 (sv/defmethod ::delete-team-member
   [{:keys [pool] :as cfg} {:keys [team-id profile-id member-id] :as params}]
   (db/with-atomic [conn pool]
-    (let [perms (teams/check-read-permissions! conn profile-id team-id)]
-      (when-not (or (some :is-owner perms)
-                    (some :is-admin perms))
+    (let [perms (teams/get-permissions conn profile-id team-id)]
+      (when-not (or (:is-owner perms)
+                    (:is-admin perms))
         (ex/raise :type :validation
                   :code :insufficient-permissions))
 
@@ -251,12 +287,13 @@
   (s/keys :req-un [::profile-id ::team-id ::file]))
 
 (sv/defmethod ::update-team-photo
+  {::rlimit/permits (cf/get :rlimit-image)}
   [{:keys [pool storage] :as cfg} {:keys [profile-id file team-id] :as params}]
   (db/with-atomic [conn pool]
     (teams/check-edition-permissions! conn profile-id team-id)
     (media/validate-media-type (:content-type file) #{"image/jpeg" "image/png" "image/webp"})
-    (media/run cfg {:cmd :info :input {:path (:tempfile file)
-                                       :mtype (:content-type file)}})
+    (media/run {:cmd :info :input {:path (:tempfile file)
+                                   :mtype (:content-type file)}})
 
     (let [team    (teams/retrieve-team conn profile-id team-id)
           storage (media/configure-assets-storage storage conn)
@@ -276,16 +313,13 @@
 
 (defn upload-photo
   [{:keys [storage] :as cfg} {:keys [file]}]
-  (let [thumb  (media/run cfg
-                 {:cmd :profile-thumbnail
-                  :format :jpeg
-                  :quality 85
-                  :width 256
-                  :height 256
-                  :input {:path (fs/path (:tempfile file))
-                          :mtype (:content-type file)}})]
-
-
+  (let [thumb (media/run {:cmd :profile-thumbnail
+                          :format :jpeg
+                          :quality 85
+                          :width 256
+                          :height 256
+                          :input {:path (fs/path (:tempfile file))
+                                  :mtype (:content-type file)}})]
     (sto/put-object storage
                     {:content (sto/content (:data thumb) (:size thumb))
                      :content-type (:mtype thumb)})))
@@ -293,30 +327,20 @@
 
 ;; --- Mutation: Invite Member
 
+(declare create-team-invitation)
+
 (s/def ::email ::us/email)
 (s/def ::invite-team-member
   (s/keys :req-un [::profile-id ::team-id ::email ::role]))
 
 (sv/defmethod ::invite-team-member
-  [{:keys [pool tokens] :as cfg} {:keys [profile-id team-id email role] :as params}]
+  [{:keys [pool] :as cfg} {:keys [profile-id team-id email role] :as params}]
   (db/with-atomic [conn pool]
-    (let [perms    (teams/check-edition-permissions! conn profile-id team-id)
+    (let [perms    (teams/get-permissions conn profile-id team-id)
           profile  (db/get-by-id conn :profile profile-id)
-          member   (profile/retrieve-profile-data-by-email conn email)
-          team     (db/get-by-id conn :team team-id)
-          itoken   (tokens :generate
-                           {:iss :team-invitation
-                            :exp (dt/in-future "48h")
-                            :profile-id (:id profile)
-                            :role role
-                            :team-id team-id
-                            :member-email (:email member email)
-                            :member-id (:id member)})
-          ptoken   (tokens :generate-predefined
-                           {:iss :profile-identity
-                            :profile-id (:id profile)})]
+          team     (db/get-by-id conn :team team-id)]
 
-      (when-not (some :is-admin perms)
+      (when-not (:is-admin perms)
         (ex/raise :type :validation
                   :code :insufficient-permissions))
 
@@ -326,24 +350,78 @@
                   :code :profile-is-muted
                   :hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
 
-      (when (and member (not (eml/allow-send-emails? conn member)))
-        (ex/raise :type :validation
-                  :code :member-is-muted
-                  :hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
-
-      ;; Secondly check if the invited member email is part of the
-      ;; global spam/bounce report.
-      (when (eml/has-bounce-reports? conn email)
-        (ex/raise :type :validation
-                  :code :email-has-permanent-bounces
-                  :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
-
-      (eml/send! {::eml/conn conn
-                  ::eml/factory eml/invite-to-team
-                  :public-uri (:public-uri cfg)
-                  :to email
-                  :invited-by (:fullname profile)
-                  :team (:name team)
-                  :token itoken
-                  :extra-data ptoken})
+      (create-team-invitation
+       (assoc cfg
+              :email email
+              :conn conn
+              :team team
+              :profile profile
+              :role role))
       nil)))
+
+(defn- create-team-invitation
+  [{:keys [conn tokens team profile role email] :as cfg}]
+  (let [member   (profile/retrieve-profile-data-by-email conn email)
+        itoken   (tokens :generate
+                         {:iss :team-invitation
+                          :exp (dt/in-future "48h")
+                          :profile-id (:id profile)
+                          :role role
+                          :team-id (:id team)
+                          :member-email (:email member email)
+                          :member-id (:id member)})
+        ptoken   (tokens :generate-predefined
+                         {:iss :profile-identity
+                          :profile-id (:id profile)})]
+
+    (when (and member (not (eml/allow-send-emails? conn member)))
+      (ex/raise :type :validation
+                :code :member-is-muted
+                :hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
+
+    ;; Secondly check if the invited member email is part of the global spam/bounce report.
+    (when (eml/has-bounce-reports? conn email)
+      (ex/raise :type :validation
+                :code :email-has-permanent-bounces
+                :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
+
+    (eml/send! {::eml/conn conn
+                ::eml/factory eml/invite-to-team
+                :public-uri (:public-uri cfg)
+                :to email
+                :invited-by (:fullname profile)
+                :team (:name team)
+                :token itoken
+                :extra-data ptoken})))
+
+
+;; --- Mutation: Create Team & Invite Members
+
+(s/def ::emails ::us/set-of-emails)
+(s/def ::create-team-and-invite-members
+  (s/and ::create-team (s/keys :req-un [::emails ::role])))
+
+(sv/defmethod ::create-team-and-invite-members
+  [{:keys [pool audit] :as cfg} {:keys [profile-id emails role] :as params}]
+  (db/with-atomic [conn pool]
+    (let [team    (create-team conn params)
+          profile (db/get-by-id conn :profile profile-id)]
+
+      ;; Create invitations for all provided emails.
+      (doseq [email emails]
+        (audit :cmd :submit
+               :type "mutation"
+               :name "create-team-invitation"
+               :profile-id profile-id
+               :props {:email email
+                       :role role
+                       :profile-id profile-id})
+
+        (create-team-invitation
+         (assoc cfg
+                :conn conn
+                :team team
+                :profile profile
+                :email email
+                :role role)))
+      team)))

backend/src/app/rpc/mutations/verify_token.clj

@@ -9,6 +9,7 @@
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.db :as db]
+   [app.loggers.audit :as audit]
    [app.rpc.mutations.teams :as teams]
    [app.rpc.queries.profile :as profile]
    [app.util.services :as sv]
@@ -32,20 +33,18 @@
   (when (profile/retrieve-profile-data-by-email conn email)
     (ex/raise :type :validation
               :code :email-already-exists))
+
   (db/update! conn :profile
               {:email email}
               {:id profile-id})
-  claims)
 
-(defn- annotate-profile-activation
-  "A helper for properly increase the profile-activation metric once the
-  transaction is completed."
-  [metrics]
-  (fn []
-    ((get-in metrics [:definitions :profile-activation]) :inc)))
+  (with-meta claims
+    {::audit/name "update-profile-email"
+     ::audit/props {:email email}
+     ::audit/profile-id profile-id}))
 
 (defmethod process-token :verify-email
-  [{:keys [conn session metrics] :as cfg} _ {:keys [profile-id] :as claims}]
+  [{:keys [conn session] :as cfg} _ {:keys [profile-id] :as claims}]
   (let [profile (profile/retrieve-profile conn profile-id)
         claims  (assoc claims :profile profile)]
 
@@ -61,7 +60,9 @@
 
     (with-meta claims
       {:transform-response ((:create session) profile-id)
-       :before-complete (annotate-profile-activation metrics)})))
+       ::audit/name "verify-profile-email"
+       ::audit/props (audit/profile->props profile)
+       ::audit/profile-id (:id profile)})))
 
 (defmethod process-token :auth
   [{:keys [conn] :as cfg} _params {:keys [profile-id] :as claims}]
@@ -107,61 +108,39 @@
     (assoc member :is-active true)))
 
 (defmethod process-token :team-invitation
-  [{:keys [session] :as cfg} {:keys [profile-id token]} {:keys [member-id] :as claims}]
+  [cfg {:keys [profile-id token]} {:keys [member-id] :as claims}]
   (us/assert ::team-invitation-claims claims)
   (cond
     ;; This happens when token is filled with member-id and current
-    ;; user is already logged in with some account.
-    (and (uuid? profile-id)
-         (uuid? member-id))
-    (do
-      (accept-invitation cfg claims)
-      (if (= member-id profile-id)
-        ;; If the current session is already matches the invited
-        ;; member, then just return the token and leave the frontend
-        ;; app redirect to correct team.
-        (assoc claims :state :created)
-
-        ;; If the session does not matches the invited member, replace
-        ;; the session with a new one matching the invited member.
-        ;; This techinique should be considered secure because the
-        ;; user clicking the link he already has access to the email
-        ;; account.
-        (with-meta
-          (assoc claims :state :created)
-          {:transform-response ((:create session) member-id)})))
-
-    ;; This happens when member-id is not filled in the invitation but
-    ;; the user already has an account (probably with other mail) and
-    ;; is already logged-in.
-    (and (uuid? profile-id)
-         (nil? member-id))
-    (do
-      (accept-invitation cfg (assoc claims :member-id profile-id))
-      (assoc claims :state :created))
-
-    ;; This happens when member-id is filled but the accessing user is
-    ;; not logged-in. In this case we proceed to accept invitation and
-    ;; leave the user logged-in.
-    (and (nil? profile-id)
-         (uuid? member-id))
-    (do
-      (accept-invitation cfg claims)
+    ;; user is already logged in with exactly invited account.
+    (and (uuid? profile-id) (uuid? member-id) (= member-id profile-id))
+    (let [profile (accept-invitation cfg claims)]
       (with-meta
         (assoc claims :state :created)
-        {:transform-response ((:create session) member-id)}))
+        {::audit/name "accept-team-invitation"
+         ::audit/props (merge
+                        (audit/profile->props profile)
+                        {:team-id (:team-id claims)
+                         :role (:role claims)})
+         ::audit/profile-id member-id}))
 
-    ;; In this case, we wait until frontend app redirect user to
-    ;; registeration page, the user is correctly registered and the
-    ;; register mutation call us again with the same token to finally
-    ;; create the corresponding team-profile relation from the first
-    ;; condition of this if.
+    ;; This case means that invitation token does not match with
+    ;; registred user, so we need to indicate to frontend to redirect
+    ;; it to register page.
+    (nil? member-id)
+    {:invitation-token token
+     :iss :team-invitation
+     :redirect-to :auth-register
+     :state :pending}
+
+    ;; In all other cases, just tell to fontend to redirect the user
+    ;; to the login page.
     :else
     {:invitation-token token
      :iss :team-invitation
+     :redirect-to :auth-login
      :state :pending}))
 
-
 ;; --- Default
 
 (defmethod process-token :default

backend/src/app/rpc/mutations/viewer.clj

@@ -1,49 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.rpc.mutations.viewer
-  (:require
-   [app.common.spec :as us]
-   [app.db :as db]
-   [app.rpc.queries.files :as files]
-   [app.util.services :as sv]
-   [buddy.core.codecs :as bc]
-   [buddy.core.nonce :as bn]
-   [clojure.spec.alpha :as s]))
-
-(s/def ::profile-id ::us/uuid)
-(s/def ::file-id ::us/uuid)
-(s/def ::page-id ::us/uuid)
-
-(s/def ::create-file-share-token
-  (s/keys :req-un [::profile-id ::file-id ::page-id]))
-
-(sv/defmethod ::create-file-share-token
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id page-id] :as params}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (let [token (-> (bn/random-bytes 16)
-                    (bc/bytes->b64u)
-                    (bc/bytes->str))]
-      (db/insert! conn :file-share-token
-                  {:file-id file-id
-                   :page-id page-id
-                   :token token})
-      {:token token})))
-
-
-(s/def ::token ::us/not-empty-string)
-(s/def ::delete-file-share-token
-  (s/keys :req-un [::profile-id ::file-id ::token]))
-
-(sv/defmethod ::delete-file-share-token
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id token]}]
-  (db/with-atomic [conn pool]
-    (files/check-edition-permissions! conn profile-id file-id)
-    (db/delete! conn :file-share-token
-                {:file-id file-id
-                 :token token})
-    nil))

backend/src/app/rpc/permissions.clj

@@ -37,28 +37,28 @@
            :is-admin false
            :can-edit false)))
 
-(defn make-edition-check-fn
-  "A simple factory for edition permission check functions."
+(defn make-edition-predicate-fn
+  "A simple factory for edition permission predicate functions."
   [qfn]
   (us/assert fn? qfn)
-  (fn [& args]
-    (let [rows (apply qfn args)]
-      (if (or (empty? rows)
-              (not (or (some :can-edit rows)
-                       (some :is-admin rows)
-                       (some :is-owner rows))))
-        (ex/raise :type :not-found
-                  :code :object-not-found
-                  :hint "not found")
-        rows))))
+  (fn check
+    ([perms] (:can-edit perms))
+    ([conn & args] (check (apply qfn conn args)))))
 
-(defn make-read-check-fn
-  "A simple factory for read permission check functions."
+(defn make-read-predicate-fn
+  "A simple factory for read permission predicate functions."
   [qfn]
   (us/assert fn? qfn)
+  (fn check
+    ([perms] (:can-read perms))
+    ([conn & args] (check (apply qfn conn args)))))
+
+(defn make-check-fn
+  "Helper that converts a predicate permission function to a check
+  function (function that raises an exception)."
+  [pred]
   (fn [& args]
-    (let [rows (apply qfn args)]
-      (if-not (seq rows)
-        (ex/raise :type :not-found
-                  :code :object-not-found)
-        rows))))
+    (when-not (apply pred args)
+      (ex/raise :type :not-found
+                :code :object-not-found
+                :hint "not found"))))

backend/src/app/rpc/queries/files.clj

@@ -6,12 +6,15 @@
 
 (ns app.rpc.queries.files
   (:require
+   [app.common.data :as d]
+   [app.common.pages.helpers :as cph]
    [app.common.pages.migrations :as pmg]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.rpc.permissions :as perms]
    [app.rpc.queries.projects :as projects]
+   [app.rpc.queries.share-link :refer [retrieve-share-link]]
    [app.rpc.queries.teams :as teams]
    [app.storage.impl :as simpl]
    [app.util.blob :as blob]
@@ -23,6 +26,7 @@
 
 ;; --- Helpers & Specs
 
+(s/def ::frame-id ::us/uuid)
 (s/def ::id ::us/uuid)
 (s/def ::name ::us/string)
 (s/def ::project-id ::us/uuid)
@@ -59,19 +63,51 @@
     where f.id = ?
       and ppr.profile_id = ?")
 
-(defn- retrieve-file-permissions
+(defn retrieve-file-permissions
   [conn profile-id file-id]
-  (db/exec! conn [sql:file-permissions
-                  file-id profile-id
-                  file-id profile-id
-                  file-id profile-id]))
+  (when (and profile-id file-id)
+    (db/exec! conn [sql:file-permissions
+                    file-id profile-id
+                    file-id profile-id
+                    file-id profile-id])))
+
+(defn get-permissions
+  ([conn profile-id file-id]
+   (let [rows     (retrieve-file-permissions conn profile-id file-id)
+         is-owner (boolean (some :is-owner rows))
+         is-admin (boolean (some :is-admin rows))
+         can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:type :membership
+        :is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true})))
+  ([conn profile-id file-id share-id]
+   (let [perms  (get-permissions conn profile-id file-id)
+         ldata  (retrieve-share-link conn file-id share-id)]
+
+     ;; NOTE: in a future when share-link becomes more powerful and
+     ;; will allow us specify which parts of the app is available, we
+     ;; will probably need to tweak this function in order to expose
+     ;; this flags to the frontend.
+     (cond
+       (some? perms) perms
+       (some? ldata) {:type :share-link
+                      :can-read true
+                      :flags (:flags ldata)}))))
+
+(def has-edit-permissions?
+  (perms/make-edition-predicate-fn get-permissions))
+
+(def has-read-permissions?
+  (perms/make-read-predicate-fn get-permissions))
 
 (def check-edition-permissions!
-  (perms/make-edition-check-fn retrieve-file-permissions))
+  (perms/make-check-fn has-edit-permissions?))
 
 (def check-read-permissions!
-  (perms/make-read-check-fn retrieve-file-permissions))
-
+  (perms/make-check-fn has-read-permissions?))
 
 ;; --- Query: Files search
 
@@ -124,29 +160,6 @@
                     profile-id team-id
                     search-term])))
 
-
-;; --- Query: Files
-
-;; DEPRECATED: should be removed probably on 1.6.x
-
-(def ^:private sql:files
-  "select f.*
-     from file as f
-    where f.project_id = ?
-      and f.deleted_at is null
-    order by f.modified_at desc")
-
-(s/def ::project-id ::us/uuid)
-(s/def ::files
-  (s/keys :req-un [::profile-id ::project-id]))
-
-(sv/defmethod ::files
-  [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}]
-  (with-open [conn (db/open pool)]
-    (projects/check-read-permissions! conn profile-id project-id)
-    (into [] decode-row-xf (db/exec! conn [sql:files project-id]))))
-
-
 ;; --- Query: Project Files
 
 (def ^:private sql:project-files
@@ -155,6 +168,7 @@
           f.created_at,
           f.modified_at,
           f.name,
+          f.revn,
           f.is_shared
      from file as f
     where f.project_id = ?
@@ -194,17 +208,73 @@
   (s/keys :req-un [::profile-id ::id]))
 
 (sv/defmethod ::file
+  "Retrieve a file by its ID. Only authenticated users."
   [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
   (db/with-atomic [conn pool]
-    (let [cfg (assoc cfg :conn conn)]
-      (check-edition-permissions! conn profile-id id)
-      (retrieve-file cfg id))))
+    (let [cfg   (assoc cfg :conn conn)
+          perms (get-permissions conn profile-id id)]
+
+      (check-read-permissions! perms)
+      (some-> (retrieve-file cfg id)
+              (assoc :permissions perms)))))
+
+(declare trim-file-data)
+
+(s/def ::page-id ::us/uuid)
+(s/def ::object-id ::us/uuid)
+
+(s/def ::trimmed-file
+  (s/keys :req-un [::profile-id ::id ::object-id ::page-id]))
+
+(sv/defmethod ::trimmed-file
+  "Retrieve a file by its ID and trims all unnecesary content from
+  it. It is mainly used for rendering a concrete object, so we don't
+  need force download all shapes when only a small subset is
+  necesseary."
+  [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [cfg   (assoc cfg :conn conn)
+          perms (get-permissions conn profile-id id)]
+      (check-read-permissions! perms)
+      (some-> (retrieve-file cfg id)
+              (trim-file-data params)
+              (assoc :permissions perms)))))
+
+(defn- trim-file-data
+  [file {:keys [page-id object-id]}]
+  (let [page    (get-in file [:data :pages-index page-id])
+        objects (->> (cph/get-children-with-self (:objects page) object-id)
+                     (map #(dissoc % :thumbnail))
+                     (d/index-by :id))
+        page    (assoc page :objects objects)]
+    (-> file
+        (update :data assoc :pages-index {page-id page})
+        (update :data assoc :pages [page-id]))))
+
+(declare strip-frames-with-thumbnails)
+
+(s/def ::strip-frames-with-thumbnails ::us/boolean)
 
 (s/def ::page
-  (s/keys :req-un [::profile-id ::file-id]))
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::strip-frames-with-thumbnails]))
 
-(defn remove-thumbnails-frames
-  "Removes from data the children for frames that have a thumbnail set up"
+(sv/defmethod ::page
+  "Retrieves the first page of the file. Used mainly for render
+  thumbnails on dashboard."
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as props}]
+  (db/with-atomic [conn pool]
+    (check-read-permissions! conn profile-id file-id)
+
+    (let [cfg     (assoc cfg :conn conn)
+          file    (retrieve-file cfg file-id)
+          page-id (get-in file [:data :pages 0])]
+      (cond-> (get-in file [:data :pages-index page-id])
+        (true? (:strip-frames-with-thumbnails props))
+        (strip-frames-with-thumbnails)))))
+
+(defn strip-frames-with-thumbnails
+  "Remove unnecesary shapes from frames that have thumbnail."
   [data]
   (let [filter-shape?
         (fn [objects [id shape]]
@@ -213,7 +283,7 @@
                 (= frame-id uuid/zero)
                 (not (some? (get-in objects [frame-id :thumbnail]))))))
 
-        ;; We need to remove from the attribute :shapes its childrens because
+        ;; We need to remove from the attribute :shapes its children because
         ;; they will not be sent in the data
         remove-frame-children
         (fn [[id shape]]
@@ -230,43 +300,12 @@
 
     (update data :objects update-objects)))
 
-(sv/defmethod ::page
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id strip-thumbnails]}]
-  (db/with-atomic [conn pool]
-    (check-edition-permissions! conn profile-id file-id)
-    (let [cfg     (assoc cfg :conn conn)
-          file    (retrieve-file cfg file-id)
-          page-id (get-in file [:data :pages 0])]
-      (cond-> (get-in file [:data :pages-index page-id])
-        strip-thumbnails
-        (remove-thumbnails-frames)))))
-
-;; --- Query: Shared Library Files
-
-;; DEPRECATED: and will be removed on 1.6.x
-
-(def ^:private sql:shared-files
-  "select f.*
-     from file as f
-    inner join project as p on (p.id = f.project_id)
-    where f.is_shared = true
-      and f.deleted_at is null
-      and p.deleted_at is null
-      and p.team_id = ?
-    order by f.modified_at desc")
-
-(s/def ::shared-files
-  (s/keys :req-un [::profile-id ::team-id]))
-
-(sv/defmethod ::shared-files
-  [{:keys [pool] :as cfg} {:keys [team-id] :as params}]
-  (into [] decode-row-xf (db/exec! pool [sql:shared-files team-id])))
-
 
 ;; --- Query: Shared Library Files
 
 (def ^:private sql:team-shared-files
   "select f.id,
+          f.revn,
           f.project_id,
           f.created_at,
           f.modified_at,
@@ -329,7 +368,7 @@
   [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
   (db/with-atomic [conn pool]
     (let [cfg (assoc cfg :conn conn)]
-      (check-edition-permissions! conn profile-id file-id)
+      (check-read-permissions! conn profile-id file-id)
       (retrieve-file-libraries cfg false file-id))))
 
 ;; --- QUERY: team-recent-files
@@ -337,6 +376,7 @@
 (def sql:team-recent-files
   "with recent_files as (
      select f.id,
+            f.revn,
             f.project_id,
             f.created_at,
             f.modified_at,
@@ -353,6 +393,7 @@
    )
    select * from recent_files where row_num <= 10;")
 
+
 (s/def ::team-recent-files
   (s/keys :req-un [::profile-id ::team-id]))
 
@@ -362,6 +403,25 @@
     (teams/check-read-permissions! conn profile-id team-id)
     (db/exec! conn [sql:team-recent-files team-id])))
 
+
+;; --- QUERY: get the thumbnail for an frame
+
+(def ^:private sql:file-frame-thumbnail
+  "select data
+     from file_frame_thumbnail
+    where file_id = ?
+      and frame_id = ?")
+
+(s/def ::file-frame-thumbnail
+  (s/keys :req-un [::profile-id ::file-id ::frame-id]))
+
+(sv/defmethod ::file-frame-thumbnail
+  [{:keys [pool]} {:keys [profile-id file-id frame-id]}]
+  (with-open [conn (db/open pool)]
+    (check-read-permissions! conn profile-id file-id)
+    (db/exec-one! conn [sql:file-frame-thumbnail file-id frame-id])))
+
+
 ;; --- Helpers
 
 (defn decode-row

backend/src/app/rpc/queries/profile.clj

@@ -35,12 +35,17 @@
 (s/def ::profile
   (s/keys :opt-un [::profile-id]))
 
-(sv/defmethod ::profile {:auth false}
+(sv/defmethod ::profile
+  {:auth false}
   [{:keys [pool] :as cfg} {:keys [profile-id] :as params}]
-  (if profile-id
-    (retrieve-profile pool profile-id)
-    {:id uuid/zero
-     :fullname "Anonymous User"}))
+  ;; We need to return the anonymous profile object in two cases, when
+  ;; no profile-id is in session, and when db call raises not found. In all other
+  ;; cases we need to reraise the exception.
+  (or (ex/try*
+       #(some->> profile-id (retrieve-profile pool))
+       #(when (not= :not-found (:type (ex-data %))) (throw %)))
+      {:id uuid/zero
+       :fullname "Anonymous User"}))
 
 (def ^:private sql:default-profile-team
   "select t.id, name
@@ -70,6 +75,10 @@
   [conn profile]
   (merge profile (retrieve-additional-data conn (:id profile))))
 
+(defn filter-profile-props
+  [props]
+  (into {} (filter (fn [[k _]] (simple-ident? k))) props))
+
 (defn decode-profile-row
   [{:keys [props] :as row}]
   (cond-> row
@@ -83,14 +92,10 @@
 
 (defn retrieve-profile
   [conn id]
-  (let [profile (some->> (retrieve-profile-data conn id)
-                         (strip-private-attrs)
-                         (populate-additional-data conn))]
-    (when (nil? profile)
-      (ex/raise :type :not-found
-                :hint "Object doest not exists."))
-
-    profile))
+  (let [profile (->> (retrieve-profile-data conn id)
+                     (strip-private-attrs)
+                     (populate-additional-data conn))]
+    (update profile :props filter-profile-props)))
 
 (def ^:private sql:profile-by-email
   "select p.* from profile as p
@@ -106,6 +111,6 @@
 ;; --- Attrs Helpers
 
 (defn strip-private-attrs
-  "Only selects a publicy visible profile attrs."
+  "Only selects a publicly visible profile attrs."
   [row]
   (dissoc row :password :deleted-at))

backend/src/app/rpc/queries/projects.clj

@@ -31,18 +31,31 @@
     where ppr.project_id = ?
       and ppr.profile_id = ?")
 
-(defn- retrieve-project-permissions
+(defn- get-permissions
   [conn profile-id project-id]
-  (db/exec! conn [sql:project-permissions
-                  project-id profile-id
-                  project-id profile-id]))
+  (let [rows     (db/exec! conn [sql:project-permissions
+                                 project-id profile-id
+                                 project-id profile-id])
+        is-owner (boolean (some :is-owner rows))
+        is-admin (boolean (some :is-admin rows))
+        can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true})))
+
+(def has-edit-permissions?
+  (perms/make-edition-predicate-fn get-permissions))
+
+(def has-read-permissions?
+  (perms/make-read-predicate-fn get-permissions))
 
 (def check-edition-permissions!
-  (perms/make-edition-check-fn retrieve-project-permissions))
+  (perms/make-check-fn has-edit-permissions?))
 
 (def check-read-permissions!
-  (perms/make-read-check-fn retrieve-project-permissions))
-
+  (perms/make-check-fn has-read-permissions?))
 
 ;; --- Query: Projects
 
@@ -66,12 +79,14 @@
             where f.project_id = p.id
               and deleted_at is null) as count
      from project as p
+    inner join team as t on (t.id = p.team_id)
      left join team_project_profile_rel as tpp
             on (tpp.project_id = p.id and
                 tpp.team_id = p.team_id and
                 tpp.profile_id = ?)
     where p.team_id = ?
       and p.deleted_at is null
+      and t.deleted_at is null
     order by p.modified_at desc")
 
 (defn retrieve-projects
@@ -95,26 +110,26 @@
 (def sql:all-projects
   "select p1.*, t.name as team_name, t.is_default as is_default_team
      from project as p1
-    inner join team as t
-            on t.id = p1.team_id
+    inner join team as t on (t.id = p1.team_id)
     where t.id in (select team_id
                      from team_profile_rel as tpr
                     where tpr.profile_id = ?
                       and (tpr.can_edit = true or
                            tpr.is_owner = true or
                            tpr.is_admin = true))
+      and t.deleted_at is null
       and p1.deleted_at is null
    union
    select p2.*, t.name as team_name, t.is_default as is_default_team
      from project as p2
-    inner join team as t
-              on t.id = p2.team_id
+    inner join team as t on (t.id = p2.team_id)
     where p2.id in (select project_id
                      from project_profile_rel as ppr
                     where ppr.profile_id = ?
                       and (ppr.can_edit = true or
                            ppr.is_owner = true or
                            ppr.is_admin = true))
+      and t.deleted_at is null
       and p2.deleted_at is null
     order by team_name, name;")
 

backend/src/app/rpc/queries/recent_files.clj

@@ -1,42 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.rpc.queries.recent-files
-  (:require
-   [app.common.spec :as us]
-   [app.db :as db]
-   [app.rpc.queries.files :refer [decode-row-xf]]
-   [app.rpc.queries.teams :as teams]
-   [app.util.services :as sv]
-   [clojure.spec.alpha :as s]))
-
-;; DEPRECATED: should be removed on 1.6.x
-
-(def sql:recent-files
-  "with recent_files as (
-     select f.*, row_number() over w as row_num
-       from file as f
-       join project as p on (p.id = f.project_id)
-      where p.team_id = ?
-        and p.deleted_at is null
-        and f.deleted_at is null
-     window w as (partition by f.project_id order by f.modified_at desc)
-      order by f.modified_at desc
-   )
-   select * from recent_files where row_num <= 10;")
-
-(s/def ::team-id ::us/uuid)
-(s/def ::profile-id ::us/uuid)
-
-(s/def ::recent-files
-  (s/keys :req-un [::profile-id ::team-id]))
-
-(sv/defmethod ::recent-files
-  [{:keys [pool] :as cfg} {:keys [profile-id team-id]}]
-  (with-open [conn (db/open pool)]
-    (teams/check-read-permissions! conn profile-id team-id)
-    (let [files (db/exec! conn [sql:recent-files team-id])]
-      (into [] decode-row-xf files))))

backend/src/app/rpc/queries/share_link.clj

@@ -0,0 +1,23 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.queries.share-link
+  (:require
+   [app.db :as db]))
+
+(defn decode-share-link-row
+  [row]
+  (-> row
+      (update :flags db/decode-pgarray #{})
+      (update :pages db/decode-pgarray #{})))
+
+(defn retrieve-share-link
+  [conn file-id share-id]
+  (some-> (db/get-by-params conn :share-link
+                            {:id share-id :file-id file-id}
+                            {:check-not-found false})
+          (decode-share-link-row)))
+

backend/src/app/rpc/queries/teams.clj

@@ -21,19 +21,34 @@
           tpr.is_admin,
           tpr.can_edit
      from team_profile_rel as tpr
+     join team as t on (t.id = tpr.team_id)
     where tpr.profile_id = ?
-      and tpr.team_id = ?")
+      and tpr.team_id = ?
+      and t.deleted_at is null")
 
-(defn- retrieve-team-permissions
+(defn get-permissions
   [conn profile-id team-id]
-  (db/exec! conn [sql:team-permissions profile-id team-id]))
+  (let [rows     (db/exec! conn [sql:team-permissions profile-id team-id])
+        is-owner (boolean (some :is-owner rows))
+        is-admin (boolean (some :is-admin rows))
+        can-edit (boolean (some :can-edit rows))]
+     (when (seq rows)
+       {:is-owner is-owner
+        :is-admin (or is-owner is-admin)
+        :can-edit (or is-owner is-admin can-edit)
+        :can-read true})))
+
+(def has-edit-permissions?
+  (perms/make-edition-predicate-fn get-permissions))
+
+(def has-read-permissions?
+  (perms/make-read-predicate-fn get-permissions))
 
 (def check-edition-permissions!
-  (perms/make-edition-check-fn retrieve-team-permissions))
+  (perms/make-check-fn has-edit-permissions?))
 
 (def check-read-permissions!
-  (perms/make-read-check-fn retrieve-team-permissions))
-
+  (perms/make-check-fn has-read-permissions?))
 
 ;; --- Query: Teams
 
@@ -58,12 +73,26 @@
      join team as t on (t.id = tp.team_id)
     where t.deleted_at is null
       and tp.profile_id = ?
-    order by t.created_at asc")
+    order by tp.created_at asc")
+
+(defn process-permissions
+  [team]
+  (let [is-owner    (:is-owner team)
+        is-admin    (:is-admin team)
+        can-edit    (:can-edit team)
+        permissions {:type :membership
+                     :is-owner is-owner
+                     :is-admin (or is-owner is-admin)
+                     :can-edit (or is-owner is-admin can-edit)}]
+    (-> team
+        (dissoc :is-owner :is-admin :can-edit)
+        (assoc :permissions permissions))))
 
 (defn retrieve-teams
   [conn profile-id]
   (let [defaults (profile/retrieve-additional-data conn profile-id)]
-    (db/exec! conn [sql:teams (:default-team-id defaults) profile-id])))
+    (->> (db/exec! conn [sql:teams (:default-team-id defaults) profile-id])
+         (mapv process-permissions))))
 
 ;; --- Query: Team (by ID)
 
@@ -86,7 +115,7 @@
     (when-not result
       (ex/raise :type :not-found
                 :code :team-does-not-exist))
-    result))
+    (process-permissions result)))
 
 
 ;; --- Query: Team Members

backend/src/app/rpc/queries/viewer.clj

@@ -10,77 +10,74 @@
    [app.common.spec :as us]
    [app.db :as db]
    [app.rpc.queries.files :as files]
+   [app.rpc.queries.share-link :as slnk]
    [app.rpc.queries.teams :as teams]
    [app.util.services :as sv]
    [clojure.spec.alpha :as s]))
 
-;; --- Query: Viewer Bundle (by Page ID)
-
-(declare check-shared-token!)
-(declare retrieve-shared-token)
-
-(def ^:private
-  sql:project
-  "select p.id, p.name, p.team_id
-     from project as p
-    where p.id = ?
-      and p.deleted_at is null")
+;; --- Query: View Only Bundle
 
 (defn- retrieve-project
   [conn id]
-  (db/exec-one! conn [sql:project id]))
+  (db/get-by-id conn :project id {:columns [:id :name :team-id]}))
+
+(defn- retrieve-bundle
+  [{:keys [conn] :as cfg} file-id]
+  (let [file    (files/retrieve-file cfg file-id)
+        project (retrieve-project conn (:project-id file))
+        libs    (files/retrieve-file-libraries cfg false file-id)
+        users   (teams/retrieve-users conn (:team-id project))
+
+        links   (->> (db/query conn :share-link {:file-id file-id})
+                     (mapv slnk/decode-share-link-row))
+
+        fonts   (db/query conn :team-font-variant
+                          {:team-id (:team-id project)
+                           :deleted-at nil})]
+    {:file file
+     :users users
+     :fonts fonts
+     :project project
+     :share-links links
+     :libraries libs}))
 
-(s/def ::id ::us/uuid)
 (s/def ::file-id ::us/uuid)
-(s/def ::page-id ::us/uuid)
-(s/def ::token ::us/string)
+(s/def ::profile-id ::us/uuid)
+(s/def ::share-id ::us/uuid)
 
-(s/def ::viewer-bundle
-  (s/keys :req-un [::file-id ::page-id]
-          :opt-un [::profile-id ::token]))
+(s/def ::view-only-bundle
+  (s/keys :req-un [::file-id] :opt-un [::profile-id ::share-id]))
 
-(sv/defmethod ::viewer-bundle {:auth false}
-  [{:keys [pool] :as cfg} {:keys [profile-id file-id page-id token] :as params}]
+(sv/defmethod ::view-only-bundle {:auth false}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}]
   (db/with-atomic [conn pool]
-    (let [cfg     (assoc cfg :conn conn)
-          file    (files/retrieve-file cfg file-id)
-          project (retrieve-project conn (:project-id file))
-          page    (get-in file [:data :pages-index page-id])
-          file    (merge (dissoc file :data)
-                         (select-keys (:data file) [:colors :media :typographies]))
-          libs    (files/retrieve-file-libraries cfg false file-id)
-          users   (teams/retrieve-users conn (:team-id project))
+    (let [cfg    (assoc cfg :conn conn)
+          slink  (slnk/retrieve-share-link conn file-id share-id)
+          perms  (files/get-permissions conn profile-id file-id share-id)
 
-          fonts   (db/query conn :team-font-variant
-                            {:team-id (:team-id project)
-                             :deleted-at nil})
+          bundle (some-> (retrieve-bundle cfg file-id)
+                         (assoc :permissions perms))]
 
-          bundle  {:file file
-                   :page page
-                   :users users
-                   :fonts fonts
-                   :project project
-                   :libraries libs}]
-
-      (if (string? token)
-        (do
-          (check-shared-token! conn file-id page-id token)
-          (assoc bundle :token token))
-        (let [stoken (retrieve-shared-token conn file-id page-id)]
-          (files/check-read-permissions! conn profile-id file-id)
-          (assoc bundle :token (:token stoken)))))))
-
-(defn check-shared-token!
-  [conn file-id page-id token]
-  (let [sql "select exists(select 1 from file_share_token where file_id=? and page_id=? and token=?) as exists"]
-    (when-not (:exists (db/exec-one! conn [sql file-id page-id token]))
-      (ex/raise :type :not-found
-                :code :object-not-found))))
-
-(defn retrieve-shared-token
-  [conn file-id page-id]
-  (let [sql "select * from file_share_token where file_id=? and page_id=?"]
-    (db/exec-one! conn [sql file-id page-id])))
+      ;; When we have neither profile nor share, we just return a not
+      ;; found response to the user.
+      (when (and (not profile-id)
+                 (not slink))
+        (ex/raise :type :not-found
+                  :code :object-not-found))
 
+      ;; When we have only profile, we need to check read permissions
+      ;; on file.
+      (when (and profile-id (not slink))
+        (files/check-read-permissions! conn profile-id file-id))
 
+      (cond-> bundle
+        (some? slink)
+        (assoc :share slink)
 
+        (and (some? slink)
+             (not (contains? (:flags slink) "view-all-pages")))
+        (update-in [:file :data] (fn [data]
+                                   (let [allowed-pages (:pages slink)]
+                                     (-> data
+                                         (update :pages (fn [pages] (filterv #(contains? allowed-pages %) pages)))
+                                         (update :pages-index (fn [index] (select-keys index allowed-pages)))))))))))

backend/src/app/rpc/retry.clj

@@ -0,0 +1,45 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.retry
+  "A fault tolerance helpers. Allow retry some operations that we know
+  we can retry."
+  (:require
+   [app.common.logging :as l]
+   [app.util.services :as sv]
+   [promesa.core :as p]))
+
+(defn conflict-db-insert?
+  "Check if exception matches a insertion conflict on postgresql."
+  [e]
+  (and (instance? org.postgresql.util.PSQLException e)
+       (= "23505" (.getSQLState e))))
+
+(defn wrap-retry
+  [_ f {:keys [::matches ::sv/name]
+        :or {matches (constantly false)}
+        :as mdata}]
+
+  (when (::enabled mdata)
+    (l/debug :hint "wrapping retry" :name name))
+
+  (if-let [max-retries (::max-retries mdata)]
+    (fn [cfg params]
+      (letfn [(run [retry]
+                (-> (f cfg params)
+                    (p/catch (partial handle-error retry))))
+
+              (handle-error [retry cause]
+                (if (matches cause)
+                  (let [current-retry (inc retry)]
+                    (l/trace :hint "running retry algorithm" :retry current-retry)
+                    (if (<= current-retry max-retries)
+                      (run current-retry)
+                      (throw cause)))
+                  (throw cause)))]
+        (run 0)))
+    f))
+

backend/src/app/rpc/rlimit.clj

@@ -0,0 +1,67 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.rlimit
+  "Resource usage limits (in other words: semaphores)."
+  (:require
+   [app.common.data :as d]
+   [app.common.logging :as l]
+   [app.metrics :as mtx]
+   [app.util.services :as sv]
+   [promesa.core :as p]))
+
+(defprotocol IAsyncSemaphore
+  (acquire! [_])
+  (release! [_]))
+
+(defn semaphore
+  [{:keys [permits metrics name]}]
+  (let [name   (d/name name)
+        used   (volatile! 0)
+        queue  (volatile! (d/queue))
+        labels (into-array String [name])]
+    (reify IAsyncSemaphore
+      (acquire! [this]
+        (let [d (p/deferred)]
+          (locking this
+            (if (< @used permits)
+              (do
+                (vswap! used inc)
+                (p/resolve! d))
+              (vswap! queue conj d)))
+
+          (mtx/run! metrics {:id :rlimit-used-permits :val @used :labels labels })
+          (mtx/run! metrics {:id :rlimit-queued-submissions :val (count @queue) :labels labels})
+          (mtx/run! metrics {:id :rlimit-acquires-total :inc 1 :labels labels})
+          d))
+
+      (release! [this]
+        (locking this
+          (if-let [item (peek @queue)]
+            (do
+              (vswap! queue pop)
+              (p/resolve! item))
+            (when (pos? @used)
+              (vswap! used dec))))
+
+        (mtx/run! metrics {:id :rlimit-used-permits :val @used :labels labels})
+        (mtx/run! metrics {:id :rlimit-queued-submissions :val (count @queue) :labels labels})
+        ))))
+
+(defn wrap-rlimit
+  [{:keys [metrics] :as cfg} f mdata]
+  (if-let [permits (::permits mdata)]
+    (let [sem (semaphore {:permits permits
+                          :metrics metrics
+                          :name (::sv/name mdata)})]
+      (l/debug :hint "wrapping rlimit" :handler (::sv/name mdata) :permits permits)
+      (fn [cfg params]
+        (-> (acquire! sem)
+            (p/then (fn [_] (f cfg params)))
+            (p/finally (fn [_ _] (release! sem))))))
+    f))
+
+

backend/src/app/setup.clj

@@ -7,6 +7,7 @@
 (ns app.setup
   "Initial data setup of instance."
   (:require
+   [app.common.logging :as l]
    [app.common.uuid :as uuid]
    [app.db :as db]
    [buddy.core.codecs :as bc]
@@ -14,55 +15,49 @@
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
 
-(declare initialize-instance-id!)
-(declare initialize-secret-key!)
-(declare retrieve-all)
+(defn- generate-random-key
+  []
+  (-> (bn/random-bytes 64)
+      (bc/bytes->b64u)
+      (bc/bytes->str)))
+
+(defn- retrieve-all
+  [conn]
+  (->> (db/query conn :server-prop {:preload true})
+       (filter #(not= "secret-key" (:id %)))
+       (map (fn [row]
+              [(keyword (:id row))
+               (db/decode-transit-pgobject (:content row))]))
+       (into {})))
+
+(defn- handle-instance-id
+  [instance-id conn read-only?]
+  (or instance-id
+      (let [instance-id (uuid/random)]
+        (when-not read-only?
+          (try
+            (db/insert! conn :server-prop
+                        {:id "instance-id"
+                         :preload true
+                         :content (db/tjson instance-id)})
+            (catch Throwable cause
+              (l/warn :hint "unable to persist instance-id"
+                      :instance-id instance-id
+                      :cause cause))))
+        instance-id)))
 
 (defmethod ig/pre-init-spec ::props [_]
   (s/keys :req-un [::db/pool]))
 
 (defmethod ig/init-key ::props
-  [_ {:keys [pool] :as cfg}]
+  [_ {:keys [pool key] :as cfg}]
   (db/with-atomic [conn pool]
-    (let [cfg (assoc cfg :conn conn)]
-      (initialize-secret-key! cfg)
-      (initialize-instance-id! cfg)
-      (retrieve-all cfg))))
+    (db/xact-lock! conn 0)
+    (when-not key
+      (l/warn :hint (str "using autogenerated secret-key, it will change on each restart and will invalidate "
+                         "all sessions on each restart, it is hightly recommeded setting up the "
+                         "PENPOT_SECRET_KEY environment variable")))
 
-(def sql:upsert-secret-key
-  "insert into server_prop (id, preload, content)
-   values ('secret-key', true, ?::jsonb)
-   on conflict (id) do update set content = ?::jsonb")
-
-(def sql:insert-secret-key
-  "insert into server_prop (id, preload, content)
-   values ('secret-key', true, ?::jsonb)
-   on conflict (id) do nothing")
-
-(defn- initialize-secret-key!
-  [{:keys [conn key] :as cfg}]
-  (if key
-    (let [key (db/tjson key)]
-      (db/exec-one! conn [sql:upsert-secret-key key key]))
-    (let [key (-> (bn/random-bytes 64)
-                  (bc/bytes->b64u)
-                  (bc/bytes->str))
-          key (db/tjson key)]
-      (db/exec-one! conn [sql:insert-secret-key key]))))
-
-(defn- initialize-instance-id!
-  [{:keys [conn] :as cfg}]
-  (let [iid (uuid/random)]
-
-    (db/insert! conn :server-prop
-                {:id "instance-id"
-                 :preload true
-                 :content (db/tjson iid)}
-                {:on-conflict-do-nothing true})))
-
-(defn- retrieve-all
-  [{:keys [conn] :as cfg}]
-  (reduce (fn [acc row]
-            (assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row))))
-          {}
-          (db/query conn :server-prop {:preload true})))
+    (let [stored (-> (retrieve-all conn)
+                     (assoc :secret-key (or key (generate-random-key))))]
+      (update stored :instance-id handle-instance-id conn (db/read-only? pool)))))

backend/src/app/srepl.clj

@@ -7,9 +7,9 @@
 (ns app.srepl
   "Server Repl."
   (:require
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.srepl.main]
-   [app.util.logging :as l]
    [clojure.core.server :as ccs]
    [clojure.main :as cm]
    [clojure.spec.alpha :as s]

backend/src/app/srepl/main.clj

@@ -2,9 +2,13 @@
   "A  main namespace for server repl."
   #_:clj-kondo/ignore
   (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.pages :as cp]
-   [app.common.uuid :as uuid]
    [app.common.pages.migrations :as pmg]
+   [app.common.spec.file :as spec.file]
+   [app.common.uuid :as uuid]
    [app.config :as cfg]
    [app.db :as db]
    [app.db.sql :as sql]
@@ -12,12 +16,15 @@
    [app.rpc.queries.profile :as prof]
    [app.srepl.dev :as dev]
    [app.util.blob :as blob]
+   [app.util.time :as dt]
+   [fipp.edn :refer [pprint]]
+   [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [clojure.pprint :refer [pprint]]))
+   [expound.alpha :as expound]))
 
 (defn update-file
-  ([id f] (update-file id f false))
-  ([id f save?]
+  ([system id f] (update-file system id f false))
+  ([system id f save?]
    (db/with-atomic [conn (:app.db/pool system)]
      (let [file (db/get-by-id conn :file id {:for-update true})
            file (-> file
@@ -32,8 +39,8 @@
                      {:id (:id file)}))
        (update file :data blob/decode)))))
 
-(defn update-file-raw
-  [id data]
+(defn reset-file-data
+  [system id data]
   (db/with-atomic [conn (:app.db/pool system)]
     (db/update! conn :file
                 {:data data}
@@ -41,36 +48,13 @@
 
 (defn get-file
   [system id]
-  (with-open [conn (db/open (:app.db/pool system))]
-    (let [file (db/get-by-id conn :file id)]
-      (-> file
-          (update :data app.util.blob/decode)
-          (update :data pmg/migrate-data)))))
-
-
-;; Examples:
-;; (def backup (update-file  #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513" identity))
-;; (def x (update-file
-;;         #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513"
-;;         (fn [{:keys [data] :as file}]
-;;           (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil))))
-
-;; Migrate
-
-(defn update-file-data-blob-format
-  [system]
-  (db/with-atomic [conn (:app.db/pool system)]
-    (doseq [id (->> (db/exec! conn ["select id from file;"]) (map :id))]
-      (let [{:keys [data]} (db/get-by-id conn :file id {:columns [:id :data]})]
-        (prn "Updating file:" id)
-        (db/update! conn :file
-                    {:data (-> (blob/decode data)
-                               (blob/encode {:version 2}))}
-                    {:id id})))))
-
+  (-> (:app.db/pool system)
+      (db/get-by-id :file id)
+      (update :data app.util.blob/decode)
+      (update :data pmg/migrate-data)))
 
 (defn duplicate-file
-  "This is a raw version of duplication of file just only for forensic analisys"
+  "This is a raw version of duplication of file just only for forensic analysis"
   [system file-id email]
   (db/with-atomic [conn (:app.db/pool system)]
     (when-let [profile (some->> (prof/retrieve-profile-data-by-email conn (str/lower email))
@@ -81,3 +65,87 @@
                             :project-id (:default-project-id profile))]
           (db/insert! conn :file params)
           (:id file))))))
+
+(defn verify-files
+  [system {:keys [age sleep chunk-size max-chunks stop-on-error? verbose?]
+           :or {sleep 1000
+                age "72h"
+                chunk-size 10
+                verbose? false
+                stop-on-error? true
+                max-chunks ##Inf}}]
+
+  (letfn [(retrieve-chunk [conn cursor]
+            (let [sql (str "select id, name, modified_at, data from file "
+                           " where modified_at > ? and deleted_at is null "
+                           " order by modified_at asc limit ?")
+                  age (if cursor
+                        cursor
+                        (-> (dt/now) (dt/minus age)))]
+              (seq (db/exec! conn [sql age chunk-size]))))
+
+          (validate-item [{:keys [id data modified-at] :as file}]
+            (let [data   (blob/decode data)
+                  valid? (s/valid? ::spec.file/data data)]
+
+              (l/debug :hint "validated file"
+                       :file-id id
+                       :age (-> (dt/diff modified-at (dt/now))
+                                (dt/truncate :minutes)
+                                (str)
+                                (subs 2)
+                                (str/lower))
+                       :valid valid?)
+
+              (when (and (not valid?) verbose?)
+                (let [edata (-> (s/explain-data ::spec.file/data data)
+                                (update ::s/problems #(take 5 %)))]
+                  (binding [s/*explain-out* expound/printer]
+                    (l/warn ::l/raw (with-out-str (s/explain-out edata))))))
+
+              (when (and (not valid?) stop-on-error?)
+                (throw (ex-info "penpot/abort" {})))
+
+              valid?))
+
+          (validate-chunk [chunk]
+            (loop [items   chunk
+                   success 0
+                   errored 0]
+
+              (if-let [item (first items)]
+                (if (validate-item item)
+                  (recur (rest items) (inc success) errored)
+                  (recur (rest items) success (inc errored)))
+                [(:modified-at (last chunk))
+                 success
+                 errored])))
+
+          (fmt-result [ns ne]
+            {:total (+ ns ne)
+             :errors ne
+             :success ns})
+
+          ]
+
+    (try
+      (db/with-atomic [conn (:app.db/pool system)]
+        (loop [cursor  nil
+               chunks  0
+               success 0
+               errors 0]
+          (if (< chunks max-chunks)
+            (if-let [chunk (retrieve-chunk conn cursor)]
+              (let [[cursor success' errors'] (validate-chunk chunk)]
+                (Thread/sleep (inst-ms (dt/duration sleep)))
+                (recur cursor
+                       (inc chunks)
+                       (+ success success')
+                       (+ errors errors')))
+              (fmt-result success errors))
+            (fmt-result success errors))))
+      (catch Throwable cause
+        (when (not= "penpot/abort" (ex-message cause))
+          (throw cause))
+        :error))))
+

backend/src/app/storage.clj

@@ -9,6 +9,7 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.uuid :as uuid]
    [app.db :as db]
@@ -16,13 +17,10 @@
    [app.storage.fs :as sfs]
    [app.storage.impl :as impl]
    [app.storage.s3 :as ss3]
-   [app.util.logging :as l]
    [app.util.time :as dt]
-   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [datoteka.core :as fs]
-   [integrant.core :as ig]
-   [promesa.exec :as px]))
+   [integrant.core :as ig]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Storage Module State
@@ -40,7 +38,7 @@
                    :db ::sdb/backend))))
 
 (defmethod ig/pre-init-spec ::storage [_]
-  (s/keys :req-un [::wrk/executor ::db/pool ::backends]))
+  (s/keys :req-un [::db/pool ::backends]))
 
 (defmethod ig/prep-key ::storage
   [_ {:keys [backends] :as cfg}]
@@ -53,78 +51,74 @@
       (assoc :backends (d/without-nils backends))))
 
 (s/def ::storage
-  (s/keys :req-un [::backends ::wrk/executor ::db/pool]))
+  (s/keys :req-un [::backends ::db/pool]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Database Objects
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defrecord StorageObject [id size created-at expired-at backend])
+(defrecord StorageObject [id size created-at expired-at touched-at backend])
 
 (defn storage-object?
   [v]
   (instance? StorageObject v))
 
-(def ^:private
-  sql:insert-storage-object
-  "insert into storage_object (id, size, backend, metadata)
-   values (?, ?, ?, ?::jsonb)
-   returning *")
+(s/def ::storage-object storage-object?)
+(s/def ::storage-content impl/content?)
 
-(def ^:private
-  sql:insert-storage-object-with-expiration
-  "insert into storage_object (id, size, backend, metadata, deleted_at)
-   values (?, ?, ?, ?::jsonb, ?)
-   returning *")
 
-(defn- insert-object
-  [conn id size backend mdata expiration]
-  (if expiration
-    (db/exec-one! conn [sql:insert-storage-object-with-expiration id size backend mdata expiration])
-    (db/exec-one! conn [sql:insert-storage-object id size backend mdata])))
+(defn- clone-database-object
+  ;; If we in this condition branch, this means we come from the
+  ;; clone-object, so we just need to clone it with a new backend.
+  [{:keys [conn backend]} object]
+  (let [id     (uuid/random)
+        mdata  (meta object)
+        result (db/insert! conn :storage-object
+                           {:id id
+                            :size (:size object)
+                            :backend (name backend)
+                            :metadata (db/tjson mdata)
+                            :deleted-at (:expired-at object)
+                            :touched-at (:touched-at object)})]
+    (assoc object
+           :id (:id result)
+           :backend backend
+           :created-at (:created-at result)
+           :touched-at (:touched-at result))))
 
 (defn- create-database-object
   [{:keys [conn backend]} {:keys [content] :as object}]
-  (if (instance? StorageObject object)
-    ;; If we in this condition branch, this means we come from the
-    ;; clone-object, so we just need to clone it with a new backend.
-    (let [id     (uuid/random)
-          mdata  (meta object)
-          result (insert-object conn
-                                id
-                                (:size object)
-                                (name backend)
-                                (db/tjson mdata)
-                                (:expired-at object))]
-      (assoc object
-             :id (:id result)
-             :backend backend
-             :created-at (:created-at result)))
-    (let [id     (uuid/random)
-          mdata  (dissoc object :content :expired-at)
-          result (insert-object conn
-                                id
-                                (count content)
-                                (name backend)
-                                (db/tjson mdata)
-                                (:expired-at object))]
-      (StorageObject. (:id result)
-                      (:size result)
-                      (:created-at result)
-                      (:deleted-at result)
-                      backend
-                      mdata
-                      nil))))
+  (us/assert ::storage-content content)
+  (let [id     (uuid/random)
+        mdata  (dissoc object :content :expired-at :touched-at)
+
+        result (db/insert! conn :storage-object
+                           {:id id
+                            :size (count content)
+                            :backend (name backend)
+                            :metadata (db/tjson mdata)
+                            :deleted-at (:expired-at object)
+                            :touched-at (:touched-at object)})]
+
+    (StorageObject. (:id result)
+                    (:size result)
+                    (:created-at result)
+                    (:deleted-at result)
+                    (:touched-at result)
+                    backend
+                    mdata
+                    nil)))
 
 (def ^:private sql:retrieve-storage-object
   "select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
 
 (defn row->storage-object [res]
-  (let [mdata (some-> (:metadata res) (db/decode-transit-pgobject))]
+  (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
     (StorageObject. (:id res)
                     (:size res)
                     (:created-at res)
                     (:deleted-at res)
+                    (:touched-at res)
                     (keyword (:backend res))
                     mdata
                     nil)))
@@ -142,10 +136,6 @@
   (let [result (db/exec-one! conn [sql:delete-storage-object id])]
     (pos? (:next.jdbc/update-count result))))
 
-(defn- register-recheck
-  [{:keys [pool] :as storage} backend id]
-  (db/insert! pool :storage-pending {:id id :backend (name backend)}))
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; API
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -170,17 +160,13 @@
 
 (defn put-object
   "Creates a new object with the provided content."
-  [{:keys [pool conn backend executor] :as storage} {:keys [content] :as params}]
+  [{:keys [pool conn backend] :as storage} {:keys [content] :as params}]
   (us/assert ::storage storage)
-  (us/assert impl/content? content)
+  (us/assert ::storage-content content)
+  (us/assert ::us/keyword backend)
   (let [storage (assoc storage :conn (or conn pool))
         object  (create-database-object storage params)]
 
-    ;; Schedule to execute in background; in an other transaction and
-    ;; register the currently created storage object id for a later
-    ;; recheck.
-    (px/run! executor #(register-recheck storage backend (:id object)))
-
     ;; Store the data finally on the underlying storage subsystem.
     (-> (impl/resolve-backend storage backend)
         (impl/put-object object content))
@@ -188,12 +174,14 @@
     object))
 
 (defn clone-object
-  "Creates a clone of the provided object using backend basded efficient
+  "Creates a clone of the provided object using backend based efficient
   method. Always clones objects to the configured default."
-  [{:keys [pool conn] :as storage} object]
+  [{:keys [pool conn backend] :as storage} object]
   (us/assert ::storage storage)
+  (us/assert ::storage-object object)
+  (us/assert ::us/keyword backend)
   (let [storage (assoc storage :conn (or conn pool))
-        object* (create-database-object storage object)]
+        object* (clone-database-object storage object)]
     (if (= (:backend object) (:backend storage))
       ;; if the source and destination backends are the same, we
       ;; proceed to use the fast path with specific copy
@@ -269,7 +257,7 @@
 ;; A task responsible to permanently delete already marked as deleted
 ;; storage files.
 
-(declare sql:retrieve-deleted-objects)
+(declare sql:retrieve-deleted-objects-chunk)
 
 (s/def ::min-age ::dt/duration)
 
@@ -278,44 +266,46 @@
 
 (defmethod ig/init-key ::gc-deleted-task
   [_ {:keys [pool storage min-age] :as cfg}]
-  (letfn [(group-by-backend [rows]
-            (let [conj (fnil conj [])]
-              [(reduce (fn [acc {:keys [id backend]}]
-                         (update acc (keyword backend) conj id))
-                       {}
-                       rows)
-               (count rows)]))
+  (letfn [(retrieve-deleted-objects-chunk [conn cursor]
+            (let [min-age (db/interval min-age)
+                  rows    (db/exec! conn [sql:retrieve-deleted-objects-chunk min-age cursor])]
+              [(some-> rows peek :created-at)
+               (some->> (seq rows) (d/group-by' #(-> % :backend keyword) :id) seq)]))
 
           (retrieve-deleted-objects [conn]
-            (let [min-age (db/interval min-age)
-                  rows    (db/exec! conn [sql:retrieve-deleted-objects min-age])]
-              (some-> (seq rows) (group-by-backend))))
+            (->> (d/iteration (fn [cursor]
+                                (retrieve-deleted-objects-chunk conn cursor))
+                              :initk (dt/now)
+                              :vf second
+                              :kf first)
+                 (sequence cat)))
 
-          (delete-in-bulk [conn [backend ids]]
+          (delete-in-bulk [conn backend ids]
             (let [backend (impl/resolve-backend storage backend)
                   backend (assoc backend :conn conn)]
               (impl/del-objects-in-bulk backend ids)))]
 
     (fn [_]
       (db/with-atomic [conn pool]
-        (loop [n 0]
-          (if-let [[groups total] (retrieve-deleted-objects conn)]
+        (loop [total  0
+               groups (retrieve-deleted-objects conn)]
+          (if-let [[backend ids] (first groups)]
             (do
-              (run! (partial delete-in-bulk conn) groups)
-              (recur (+ n ^long total)))
+              (delete-in-bulk conn backend ids)
+              (recur (+ total (count ids))
+                     (rest groups)))
             (do
-              (l/info :task "gc-deleted"
-                      :action "permanently delete items"
-                      :count n)
-              {:deleted n})))))))
+              (l/info :task "gc-deleted" :count total)
+              {:deleted total})))))))
 
-(def sql:retrieve-deleted-objects
+(def sql:retrieve-deleted-objects-chunk
   "with items_part as (
      select s.id
        from storage_object as s
       where s.deleted_at is not null
         and s.deleted_at < (now() - ?::interval)
-      order by s.deleted_at
+        and s.created_at < ?
+      order by s.created_at desc
       limit 100
    )
    delete from storage_object
@@ -323,160 +313,108 @@
    returning *;")
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Garbage Collection: Analize touched objects
+;; Garbage Collection: Analyze touched objects
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; This task is part of the garbage collection of storage objects and
-;; is responsible on analizing the touched objects and mark them for deletion
-;; if corresponds.
+;; This task is part of the garbage collection of storage objects and is responsible on analyzing the touched
+;; objects and mark them for deletion if corresponds.
 ;;
-;; When file_media_object is deleted, the depending storage_object are
-;; marked as touched. This means that some files that depend on a
-;; concrete storage_object are no longer exists and maybe this
-;; storage_object is no longer necessary and can be ellegible for
-;; elimination. This task peridically analizes touched objects and
-;; mark them as freeze (means that has other references and the object
-;; is still valid) or deleted (no more references to this object so is
-;; ready to be deleted).
+;; For example: when file_media_object is deleted, the depending storage_object are marked as touched. This
+;; means that some files that depend on a concrete storage_object are no longer exists and maybe this
+;; storage_object is no longer necessary and can be eligible for elimination. This task periodically analyzes
+;; touched objects and mark them as freeze (means that has other references and the object is still valid) or
+;; deleted (no more references to this object so is ready to be deleted).
 
-(declare sql:retrieve-touched-objects)
+(declare sql:retrieve-touched-objects-chunk)
+(declare sql:retrieve-file-media-object-nrefs)
+(declare sql:retrieve-team-font-variant-nrefs)
 
 (defmethod ig/pre-init-spec ::gc-touched-task [_]
   (s/keys :req-un [::db/pool]))
 
 (defmethod ig/init-key ::gc-touched-task
   [_ {:keys [pool] :as cfg}]
-  (letfn [(group-results [rows]
-            (let [conj (fnil conj [])]
-              (reduce (fn [acc {:keys [id nrefs]}]
-                        (if (pos? nrefs)
-                          (update acc :to-freeze conj id)
-                          (update acc :to-delete conj id)))
-                      {}
-                      rows)))
+  (letfn [(has-team-font-variant-nrefs? [conn id]
+            (-> (db/exec-one! conn [sql:retrieve-team-font-variant-nrefs id id id id]) :nrefs pos?))
 
-          (retrieve-touched [conn]
-            (let [rows (db/exec! conn [sql:retrieve-touched-objects])]
-              (some-> (seq rows) (group-results))))
-
-          (mark-delete-in-bulk [conn ids]
-            (db/exec-one! conn ["update storage_object set deleted_at=now(), touched_at=null where id = ANY(?)"
-                                (db/create-array conn "uuid" (into-array java.util.UUID ids))]))
+          (has-file-media-object-nrefs? [conn id]
+            (-> (db/exec-one! conn [sql:retrieve-file-media-object-nrefs id id]) :nrefs pos?))
 
           (mark-freeze-in-bulk [conn ids]
             (db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)"
-                                (db/create-array conn "uuid" (into-array java.util.UUID ids))]))]
+                                (db/create-array conn "uuid" ids)]))
+
+          (mark-delete-in-bulk [conn ids]
+            (db/exec-one! conn ["update storage_object set deleted_at=now(), touched_at=null where id = ANY(?)"
+                                (db/create-array conn "uuid" ids)]))
+
+          (retrieve-touched-chunk [conn cursor]
+            (let [rows (->> (db/exec! conn [sql:retrieve-touched-objects-chunk cursor])
+                            (mapv #(d/update-when % :metadata db/decode-transit-pgobject)))
+                  kw   (fn [o] (if (keyword? o) o (keyword o)))]
+              (when (seq rows)
+                [(-> rows peek :created-at)
+                 ;; NOTE: we use the :file-media-object as default value for backward compatibility because when we
+                 ;; deploy it we can have old backend instances running in the same time as the new one and we can
+                 ;; still have storage-objects created without reference value.  And we know that if it does not
+                 ;; have value, it means :file-media-object.
+                 (d/group-by' #(or (some-> % :metadata :reference kw) :file-media-object) :id rows)])))
+
+          (retrieve-touched [conn]
+            (->> (d/iteration (fn [cursor]
+                                (retrieve-touched-chunk conn cursor))
+                              :initk (dt/now)
+                              :vf second
+                              :kf first)
+                 (sequence cat)))
+
+          (process-objects! [conn pred-fn ids]
+            (loop [to-freeze #{}
+                   to-delete #{}
+                   ids       (seq ids)]
+              (if-let [id (first ids)]
+                (if (pred-fn conn id)
+                  (recur (conj to-freeze id) to-delete (rest ids))
+                  (recur to-freeze (conj to-delete id) (rest ids)))
+
+                (do
+                  (some->> (seq to-freeze) (mark-freeze-in-bulk conn))
+                  (some->> (seq to-delete) (mark-delete-in-bulk conn))
+                  [(count to-freeze) (count to-delete)]))))
+          ]
 
     (fn [_]
       (db/with-atomic [conn pool]
-        (loop [cntf 0
-               cntd 0]
-          (if-let [{:keys [to-delete to-freeze]} (retrieve-touched conn)]
+        (loop [to-freeze 0
+               to-delete 0
+               groups    (retrieve-touched conn)]
+          (if-let [[reference ids] (first groups)]
+            (let [[f d] (case reference
+                          :file-media-object (process-objects! conn has-file-media-object-nrefs? ids)
+                          :team-font-variant (process-objects! conn has-team-font-variant-nrefs? ids)
+                          (ex/raise :type :internal
+                                    :code :unexpected-unknown-reference
+                                    :hint (format "unknown reference %s" (pr-str reference))))]
+              (recur (+ to-freeze f)
+                     (+ to-delete d)
+                     (rest groups)))
             (do
-              (when (seq to-delete) (mark-delete-in-bulk conn to-delete))
-              (when (seq to-freeze) (mark-freeze-in-bulk conn to-freeze))
-              (recur (+ cntf (count to-freeze))
-                     (+ cntd (count to-delete))))
-            (do
-              (l/info :task "gc-touched"
-                      :action "mark freeze"
-                      :count cntf)
-              (l/info :task "gc-touched"
-                      :action "mark for deletion"
-                      :count cntd)
-              {:freeze cntf :delete cntd})))))))
+              (l/info :task "gc-touched" :to-freeze to-freeze :to-delete to-delete)
+              {:freeze to-freeze :delete to-delete})))))))
 
-(def sql:retrieve-touched-objects
-  "select so.id,
-          ((select count(*) from file_media_object where media_id = so.id) +
-           (select count(*) from file_media_object where thumbnail_id = so.id)) as nrefs
-     from storage_object as so
+(def sql:retrieve-touched-objects-chunk
+  "select so.* from storage_object as so
     where so.touched_at is not null
-    order by so.touched_at
-    limit 100;")
+      and so.created_at < ?
+    order by so.created_at desc
+    limit 500;")
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Recheck Stalled Task
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(def sql:retrieve-file-media-object-nrefs
+  "select ((select count(*) from file_media_object where media_id = ?) +
+           (select count(*) from file_media_object where thumbnail_id = ?)) as nrefs")
 
-;; Because the physical storage (filesystem, s3, ... except db) is not
-;; transactional, in some situations we can found physical object
-;; leakage. That situations happens when the transaction that writes
-;; the file aborts, leaving the file written to the underlying storage
-;; but the reference on the database is lost with the rollback.
-;;
-;; For this situations we need to write a "log" of inserted files that
-;; are checked in some time in future. If physical file exists but the
-;; database refence does not exists means that leaked file is found
-;; and is inmediatelly deleted. The responsability of this task is
-;; check that write log for possible leaked files.
-
-(def recheck-min-age (dt/duration {:hours 1}))
-
-(declare sql:retrieve-pending-to-recheck)
-(declare sql:exists-storage-object)
-
-(defmethod ig/pre-init-spec ::recheck-task [_]
-  (s/keys :req-un [::storage ::db/pool]))
-
-(defmethod ig/init-key ::recheck-task
-  [_ {:keys [pool storage] :as cfg}]
-  (letfn [(group-results [rows]
-            (let [conj (fnil conj [])]
-              (reduce (fn [acc {:keys [id exist] :as row}]
-                        (cond-> (update acc :all conj id)
-                          (false? exist)
-                          (update :to-delete conj (dissoc row :exist))))
-                      {}
-                      rows)))
-
-          (group-by-backend [rows]
-            (let [conj (fnil conj [])]
-              (reduce (fn [acc {:keys [id backend]}]
-                        (update acc (keyword backend) conj id))
-                      {}
-                      rows)))
-
-          (retrieve-pending [conn]
-            (let [rows (db/exec! conn [sql:retrieve-pending-to-recheck (db/interval recheck-min-age)])]
-              (some-> (seq rows) (group-results))))
-
-          (delete-group [conn [backend ids]]
-            (let [backend (impl/resolve-backend storage backend)
-                  backend (assoc backend :conn conn)]
-              (impl/del-objects-in-bulk backend ids)))
-
-          (delete-all [conn ids]
-            (let [ids (db/create-array conn "uuid" (into-array java.util.UUID ids))]
-              (db/exec-one! conn ["delete from storage_pending where id = ANY(?)" ids])))]
-
-    (fn [_]
-      (db/with-atomic [conn pool]
-        (loop [n 0 d 0]
-          (if-let [{:keys [all to-delete]} (retrieve-pending conn)]
-            (let [groups (group-by-backend to-delete)]
-              (run! (partial delete-group conn) groups)
-              (delete-all conn all)
-              (recur (+ n (count all))
-                     (+ d (count to-delete))))
-            (do
-              (l/info :task "recheck"
-                      :action "recheck items"
-                      :processed n
-                      :deleted n)
-              {:processed n :deleted d})))))))
-
-(def sql:retrieve-pending-to-recheck
-  "select sp.id,
-          sp.backend,
-          sp.created_at,
-          (case when count(so.id) > 0 then true
-                else false
-            end) as exist
-     from storage_pending as sp
-     left join storage_object as so
-            on (so.id = sp.id)
-    where sp.created_at < now() - ?::interval
-    group by 1,2,3
-    order by sp.created_at asc
-    limit 100")
+(def sql:retrieve-team-font-variant-nrefs
+  "select ((select count(*) from team_font_variant where woff1_file_id = ?) +
+           (select count(*) from team_font_variant where woff2_file_id = ?) +
+           (select count(*) from team_font_variant where otf_file_id = ?) +
+           (select count(*) from team_font_variant where ttf_file_id = ?)) as nrefs")

backend/src/app/storage/impl.clj

@@ -117,11 +117,11 @@
       io/IOFactory
       (make-reader [_ opts]
 	    (io/make-reader path opts))
-      (make-writer [_ opts]
+      (make-writer [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       (make-input-stream [_ opts]
         (io/make-input-stream path opts))
-      (make-output-stream [_ opts]
+      (make-output-stream [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       clojure.lang.Counted
       (count [_] size)
@@ -138,11 +138,11 @@
       io/IOFactory
       (make-reader [_ opts]
 	    (io/make-reader bais opts))
-      (make-writer [_ opts]
+      (make-writer [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
       (make-input-stream [_ opts]
         (io/make-input-stream bais opts))
-      (make-output-stream [_ opts]
+      (make-output-stream [_ _]
         (throw (UnsupportedOperationException. "not implemented")))
 
       clojure.lang.Counted
@@ -159,11 +159,11 @@
     io/IOFactory
     (make-reader [_ opts]
       (io/make-reader is opts))
-    (make-writer [_ opts]
+    (make-writer [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
     (make-input-stream [_ opts]
       (io/make-input-stream is opts))
-    (make-output-stream [_ opts]
+    (make-output-stream [_ _]
       (throw (UnsupportedOperationException. "not implemented")))
 
     clojure.lang.Counted

backend/src/app/storage/s3.clj

@@ -33,6 +33,7 @@
    software.amazon.awssdk.services.s3.model.GetObjectRequest
    software.amazon.awssdk.services.s3.model.ObjectIdentifier
    software.amazon.awssdk.services.s3.model.PutObjectRequest
+   software.amazon.awssdk.services.s3.model.S3Error
    ;; software.amazon.awssdk.services.s3.model.GetObjectResponse
    software.amazon.awssdk.services.s3.presigner.S3Presigner
    software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
@@ -55,9 +56,10 @@
 (s/def ::region #{:eu-central-1})
 (s/def ::bucket ::us/string)
 (s/def ::prefix ::us/string)
+(s/def ::endpoint ::us/string)
 
 (defmethod ig/pre-init-spec ::backend [_]
-  (s/keys :opt-un [::region ::bucket ::prefix]))
+  (s/keys :opt-un [::region ::bucket ::prefix ::endpoint]))
 
 (defmethod ig/prep-key ::backend
   [_ {:keys [prefix] :as cfg}]
@@ -118,20 +120,31 @@
 
 (defn- ^Region lookup-region
   [region]
-  (case region
-    :eu-central-1 Region/EU_CENTRAL_1))
+  (Region/of (name region)))
 
 (defn build-s3-client
-  [{:keys [region]}]
-  (.. (S3Client/builder)
-      (region (lookup-region region))
-      (build)))
+  [{:keys [region endpoint]}]
+  (if (string? endpoint)
+    (let [uri (java.net.URI. endpoint)]
+      (.. (S3Client/builder)
+          (endpointOverride uri)
+          (region (lookup-region region))
+          (build)))
+    (.. (S3Client/builder)
+        (region (lookup-region region))
+        (build))))
 
 (defn build-s3-presigner
-  [{:keys [region]}]
-  (.. (S3Presigner/builder)
-      (region (lookup-region region))
-      (build)))
+  [{:keys [region endpoint]}]
+  (if (string? endpoint)
+    (let [uri (java.net.URI. endpoint)]
+      (.. (S3Presigner/builder)
+          (endpointOverride uri)
+          (region (lookup-region region))
+          (build)))
+    (.. (S3Presigner/builder)
+        (region (lookup-region region))
+        (build))))
 
 (defn put-object
   [{:keys [client bucket prefix]} {:keys [id] :as object} content]
@@ -231,6 +244,9 @@
                              ^DeleteObjectsRequest dor)]
     (when (.hasErrors ^DeleteObjectsResponse dres)
       (let [errors (seq (.errors ^DeleteObjectsResponse dres))]
-        (ex/raise :type :s3-error
-                  :code :error-on-bulk-delete
-                  :context errors)))))
+        (ex/raise :type :internal
+                  :code :error-on-s3-bulk-delete
+                  :s3-errors (mapv (fn [^S3Error error]
+                                     {:key (.key error)
+                                      :msg (.message error)})
+                                   errors))))))

backend/src/app/tasks/delete_object.clj

@@ -1,70 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-;; TODO: DEPRECATED
-;; Should be removed in the 1.8.x
-
-(ns app.tasks.delete-object
-  "Generic task for permanent deletion of objects."
-  (:require
-   [app.common.data :as d]
-   [app.common.spec :as us]
-   [app.db :as db]
-   [app.storage :as sto]
-   [app.util.logging :as l]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
-
-(declare handle-deletion)
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool ::sto/storage]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as cfg}]
-  (fn [{:keys [props] :as task}]
-    (us/verify ::props props)
-    (db/with-atomic [conn pool]
-      (let [cfg (assoc cfg :conn conn)]
-        (handle-deletion cfg props)))))
-
-(s/def ::type ::us/keyword)
-(s/def ::id ::us/uuid)
-(s/def ::props (s/keys :req-un [::id ::type]))
-
-(defmulti handle-deletion
-  (fn [_ props] (:type props)))
-
-(defmethod handle-deletion :default
-  [_cfg {:keys [type]}]
-  (l/warn :hint "no handler found"
-          :type (d/name type)))
-
-(defmethod handle-deletion :file
-  [{:keys [conn]} {:keys [id] :as props}]
-  (let [sql "delete from file where id=? and deleted_at is not null"]
-    (db/exec-one! conn [sql id])))
-
-(defmethod handle-deletion :project
-  [{:keys [conn]} {:keys [id] :as props}]
-  (let [sql "delete from project where id=? and deleted_at is not null"]
-    (db/exec-one! conn [sql id])))
-
-(defmethod handle-deletion :team
-  [{:keys [conn]} {:keys [id] :as props}]
-  (let [sql "delete from team where id=? and deleted_at is not null"]
-    (db/exec-one! conn [sql id])))
-
-(defmethod handle-deletion :team-font-variant
-  [{:keys [conn storage]} {:keys [id] :as props}]
-  (let [font    (db/get-by-id conn :team-font-variant id {:uncheked true})
-        storage (assoc storage :conn conn)]
-    (when (:deleted-at font)
-      (db/delete! conn :team-font-variant {:id id})
-      (some->> (:woff1-file-id font) (sto/del-object storage))
-      (some->> (:woff2-file-id font) (sto/del-object storage))
-      (some->> (:otf-file-id font)   (sto/del-object storage))
-      (some->> (:ttf-file-id font)   (sto/del-object storage)))))

backend/src/app/tasks/delete_profile.clj

@@ -1,79 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.tasks.delete-profile
-  "Task for permanent deletion of profiles."
-  (:require
-   [app.common.spec :as us]
-   [app.db :as db]
-   [app.db.sql :as sql]
-   [app.util.logging :as l]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
-
-;; TODO: DEPRECATED
-;; Should be removed in the 1.8.x
-
-(declare delete-profile-data)
-
-;; --- INIT
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
-
-;; This task is responsible to permanently delete a profile with all
-;; the dependent data. As step (1) we delete all owned teams of the
-;; profile (that will cause to delete all underlying projects, files,
-;; file_media and mark to be deleted storage_object's used by team,
-;; profile and files previously deleted. Then, finally as step (2) we
-;; proceed to delete the profile row.
-;;
-;; The storage_objects marked as deleted will be deleted by the
-;; corresponding garbage collector task.
-
-(s/def ::profile-id ::us/uuid)
-(s/def ::props (s/keys :req-un [::profile-id]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as cfg}]
-  (fn [{:keys [props] :as task}]
-    (us/verify ::props props)
-    (db/with-atomic [conn pool]
-      (let [id      (:profile-id props)
-            profile (db/exec-one! conn (sql/select :profile {:id id} {:for-update true}))]
-        (if (or (:is-demo profile)
-                (:deleted-at profile))
-          (delete-profile-data conn id)
-          (l/warn :hint "profile does not match constraints for deletion"
-                  :profile-id id))))))
-
-;; --- IMPL
-
-(def ^:private sql:remove-owned-teams
-  "delete from team
-    where id in (
-      select tpr.team_id
-        from team_profile_rel as tpr
-       where tpr.is_owner is true
-         and tpr.profile_id = ?
-    )")
-
-(defn- delete-teams
-  [conn profile-id]
-  (db/exec-one! conn [sql:remove-owned-teams profile-id]))
-
-(defn delete-profile
-  [conn profile-id]
-  (db/delete! conn :profile {:id profile-id}))
-
-(defn- delete-profile-data
-  [conn profile-id]
-  (l/debug :action "delete profile"
-           :profile-id profile-id)
-  (delete-teams conn profile-id)
-  (delete-profile conn profile-id)
-  true)
-

backend/src/app/tasks/file_media_gc.clj

@@ -6,13 +6,14 @@
 
 (ns app.tasks.file-media-gc
   "A maintenance task that is responsible to purge the unused media
-  objects from files. A file is ellegible to be garbage collected
+  objects from files. A file is eligible to be garbage collected
   after some period of inactivity (the default threshold is 72h)."
   (:require
+   [app.common.logging :as l]
+   [app.common.pages.helpers :as cph]
    [app.common.pages.migrations :as pmg]
    [app.db :as db]
    [app.util.blob :as blob]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
@@ -52,6 +53,7 @@
     limit 10
     for update skip locked")
 
+
 (defn- retrieve-candidates
   [{:keys [conn max-age] :as cfg}]
   (let [interval (db/interval max-age)]
@@ -64,12 +66,11 @@
   (comp
    (map :objects)
    (mapcat vals)
-   (map (fn [{:keys [type] :as obj}]
-          (case type
-            :path (get-in obj [:fill-image :id])
-            :image (get-in obj [:metadata :id])
-            nil)))
-   (filter uuid?)))
+   (keep (fn [{:keys [type] :as obj}]
+           (case type
+             :path (get-in obj [:fill-image :id])
+             :image (get-in obj [:metadata :id])
+             nil)))))
 
 (defn- collect-used-media
   [data]
@@ -80,37 +81,59 @@
       (into collect-media-xf pages)
       (into (keys (:media data))))))
 
+(def ^:private
+  collect-frames-xf
+  (comp
+   (map :objects)
+   (mapcat vals)
+   (filter cph/frame-shape?)
+   (keep :id)))
+
+(defn- collect-frames
+  [data]
+  (let [pages (concat
+               (vals (:pages-index data))
+               (vals (:components data)))]
+    (into #{} collect-frames-xf pages)))
+
 (defn- process-file
   [{:keys [conn] :as cfg} {:keys [id data age] :as file}]
-  (let [data   (-> (blob/decode data)
-                   (assoc :id id)
-                   (pmg/migrate-data))
+  (let [data (-> (blob/decode data)
+                 (assoc :id id)
+                 (pmg/migrate-data))]
 
-        used   (collect-used-media data)
-        unused (->> (db/query conn :file-media-object {:file-id id})
-                    (remove #(contains? used (:id %))))]
+    (let [used   (collect-used-media data)
+          unused (->> (db/query conn :file-media-object {:file-id id})
+                      (remove #(contains? used (:id %))))]
 
-    (l/debug :action "processing file"
-             :id id
-             :age age
-             :to-delete (count unused))
+      (l/debug :hint "processing file"
+               :id id
+               :age age
+               :to-delete (count unused))
 
-    ;; Mark file as trimmed
-    (db/update! conn :file
-                {:has-media-trimmed true}
-                {:id id})
+      ;; Mark file as trimmed
+      (db/update! conn :file
+                  {:has-media-trimmed true}
+                  {:id id})
 
-    (doseq [mobj unused]
-      (l/debug :action "deleting media object"
-               :id (:id mobj)
-               :media-id (:media-id mobj)
-               :thumbnail-id (:thumbnail-id mobj))
+      (doseq [mobj unused]
+        (l/debug :hint "deleting media object"
+                 :id (:id mobj)
+                 :media-id (:media-id mobj)
+                 :thumbnail-id (:thumbnail-id mobj))
 
-      ;; NOTE: deleting the file-media-object in the database
-      ;; automatically marks as toched the referenced storage
-      ;; objects. The touch mechanism is needed because many files can
-      ;; point to the same storage objects and we can't just delete
-      ;; them.
-      (db/delete! conn :file-media-object {:id (:id mobj)}))
+        ;; NOTE: deleting the file-media-object in the database
+        ;; automatically marks as touched the referenced storage
+        ;; objects. The touch mechanism is needed because many files can
+        ;; point to the same storage objects and we can't just delete
+        ;; them.
+        (db/delete! conn :file-media-object {:id (:id mobj)})))
+
+    (let [sql (str "delete from file_frame_thumbnail "
+                   " where file_id = ? and not (frame_id = ANY(?))")
+          ids (->> (collect-frames data)
+                   (db/create-array conn "uuid"))]
+      ;; delete the unused frame thumbnails
+      (db/exec! conn [sql (:id file) ids]))
 
     nil))