✨ Add unique id (uuid) on each log entry

The prev approach uses clojure.pprint without any limit extensivelly
for format error context data and all this is done on the calling
thread. The clojure.pprint seems very inneficient in cpu and memory
usage on pprinting large data structures.

This is improved in the following way:

- All formatting and pretty printing is moved to logging thread,
  reducing unnecesary blocking and load on jetty http threads.
- Replace the clojure.pprint with fipp.edn that looks considerably
  faster than the first one.
- Add some safe limits on pretty printer for strip printing some
  data when the data structure is very large, very deep or both.

.circleci/config.yml

@@ -9,11 +9,11 @@ jobs:
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
       # - image: circleci/postgres:9.4
-      - image: circleci/postgres:13.1-ram
+      - image: circleci/postgres:13.3-ram
         environment:
           POSTGRES_USER: penpot_test
           POSTGRES_PASSWORD: penpot_test
-          POSTGRES_DB: penpot
+          POSTGRES_DB: penpot_test
 
       - image: circleci/redis:6.0.8
 
@@ -29,38 +29,82 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
          keys:
-           - v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}
+           - v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}-{{ checksum "common/deps.edn"}}
            # fallback to using the latest cache if no exact match is found
            - v1-dependencies-
 
-      # run lint
       - run:
-          working_directory: "./backend"
-          name: backend lint
-          command: "clj-kondo --lint src/"
-
-      # run test
-      - run:
-          working_directory: "./backend"
-          name: backend test
-          command: "clojure -M:dev:tests"
-          environment:
-            PENPOT_DATABASE_URI: "postgresql://localhost/penpot"
-            PENPOT_DATABASE_USERNAME: penpot_test
-            PENPOT_DATABASE_PASSWORD: penpot_test
-            PENPOT_REDIS_URI: "redis://localhost/1"
+          name: common lint
+          working_directory: "./common"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
+          name: frontend lint
+          working_directory: "./frontend"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
+
+      - run:
+          name: frontend styles prettier
           working_directory: "./frontend"
-          name: frontend tests
           command: |
             yarn install
-            npx shadow-cljs compile tests
+            yarn run lint-scss
+
+      - run:
+          name: backend lint
+          working_directory: "./backend"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
+
+      # run backend test
+      - run:
+          name: backend test
+          working_directory: "./backend"
+          command: "clojure -X:dev:test"
+          environment:
+            PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
+            PENPOT_TEST_DATABASE_USERNAME: penpot_test
+            PENPOT_TEST_DATABASE_PASSWORD: penpot_test
+            PENPOT_TEST_REDIS_URI: "redis://localhost/1"
+
+      - run:
+          name: frontend tests
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            clojure -M:dev:shadow-cljs compile test
+            node target/tests.js
+
+          environment:
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+
+      # - run:
+      #     working_directory: "./common"
+      #     name: common tests (cljs)
+      #     command: |
+      #       yarn install
+      #       yarn run compile-test
+      #       node target/test.js
+      # 
+      #     environment:
+      #       PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+
+      - run:
+          working_directory: "./common"
+          name: common tests (clj)
+          command: |
+            clojure -X:dev:test
+
           environment:
             PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
 
       - save_cache:
          paths:
            - ~/.m2
-         key: v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}
+         key: v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}-{{ checksum "common/deps.edn"}}
 

.clj-kondo/config.edn

@@ -1,13 +1,32 @@
-{:lint-as {potok.core/reify clojure.core/reify
-           promesa.core/let clojure.core/let
-           app.db/with-atomic clojure.core/with-open}
+{:lint-as
+ {promesa.core/let clojure.core/let
+  rumext.alpha/defc clojure.core/defn
+  rumext.alpha/fnc clojure.core/fn
+  app.common.data/export clojure.core/def
+  app.db/with-atomic clojure.core/with-open
+  app.common.logging/with-context clojure.core/do}
+
+ :hooks
+ {:analyze-call
+  {app.common.data/export hooks.export/export
+   potok.core/reify hooks.export/potok-reify
+   app.util.services/defmethod hooks.export/service-defmethod
+   }}
+
  :output
- {:exclude-files ["data_readers.clj"]}
+ {:exclude-files
+  ["data_readers.clj"
+   "app/util/perf.cljs"
+   "app/common/logging.cljc"
+   "app/common/exceptions.cljc"]}
 
  :linters
  {:unsorted-required-namespaces
   {:level :warning}
 
+  :potok/reify-type
+  {:level :error}
+
   :unresolved-namespace
   {:level :warning
    :exclude [data_readers]}
@@ -15,12 +34,12 @@
   :single-key-in
   {:level :warning}
 
+  :redundant-do
+  {:level :off}
+
   :unused-binding
   {:exclude-destructured-as true
    :exclude-destructured-keys-in-fn-args false
    }
-
-  :unresolved-symbol
-  {:exclude ['(app.util.services/defmethod)
-             ]}}}
+  }}
 

.clj-kondo/hooks/export.clj

@@ -0,0 +1,76 @@
+(ns hooks.export
+  (:require [clj-kondo.hooks-api :as api]))
+
+(defn export
+  [{:keys [:node]}]
+  (let [[_ sname] (:children node)
+        result  (api/list-node
+                 [(api/token-node (symbol "def"))
+                  (api/token-node (symbol (name (:value sname))))
+                  sname])]
+    {:node result}))
+
+(def registry (atom {}))
+
+(defn potok-reify
+  [{:keys [:node :filename] :as params}]
+  (let [[rnode rtype & other] (:children node)
+        rsym (symbol (str "event-type-" (name (:k rtype))))
+        reg  (get @registry filename #{})]
+    (when-not (:namespaced? rtype)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message "ptk/reify type should be namespaced"
+                           :type :potok/reify-type
+                           :row row
+                           :col col})))
+
+    (if (contains? reg rsym)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message (str "duplicate type: " (name (:k rtype)))
+                           :type :potok/reify-type
+                           :row row
+                           :col col}))
+      (swap! registry update filename (fnil conj #{}) rsym))
+
+    (let [result  (api/list-node
+                   (into [(api/token-node (symbol "deftype"))
+                          (api/token-node rsym)
+                          (api/vector-node [])]
+                         other))]
+      {:node result})))
+
+(defn clojure-specify
+  [{:keys [:node]}]
+  (let [[rnode rtype & other] (:children node)
+        result  (api/list-node
+                 (into [(api/token-node (symbol "extend-type"))
+                        (api/token-node (gensym (:string-value rtype)))]
+                       other))]
+    {:node result}))
+
+
+(defn service-defmethod
+  [{:keys [:node]}]
+  (let [[rnode rtype ?meta & other] (:children node)
+        rsym    (gensym (name (:k rtype)))
+        result  (api/list-node
+                 [(api/token-node (symbol "do"))
+                  (api/list-node
+                   [(api/token-node (symbol "declare"))
+                    (api/token-node rsym)])
+                  (if (= :map (:tag ?meta))
+                    (api/list-node
+                     [(api/token-node (symbol "reset-meta!"))
+                      (api/token-node rsym)
+                      ?meta])
+                    (api/list-node
+                     [(api/token-node (symbol "comment"))
+                      (api/token-node rsym)]))
+                  (api/list-node
+                   (into [(api/token-node (symbol "defmethod"))
+                          (api/token-node rsym)
+                          rtype]
+                         (cons ?meta other)))])]
+    ;; (prn "==============" rtype (into {} ?meta))
+    ;; (prn (api/sexpr result))
+    {:node result}))

.github/ISSUE_TEMPLATE/bug_report.md

@@ -38,13 +38,13 @@ If applicable, add screenshots to help explain your problem.
 -   Version (e.g. 22)
 
 **Environment (please complete the following information):**
-Specify if using demo instance or self-hosted instance.
+Specify if using SAAS (https://design.penpot.app) or self-hosted instance.
 
 If self-hosted instance, add OS and runtime information to help explain your problem.
 
 -   OS Version: (e.g. Ubuntu 16.04)
 
-Also provide Docker commands or docker-compose file if possible.
+Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 -   Docker / Docker-compose Version: (e.g. Docker version 18.03.0-ce, build 0520e24)
 -   Image (e.g. alpine)

.gitignore

@@ -1,36 +1,50 @@
-figwheel_server.log
-*jar
 *-init.clj
+*.jar
+*.penpot
+.calva
+.clj-kondo
+.cpcache
 .lein-deps-sum
 .lein-failures
-.lein-repl-history
 .lein-plugins/
-.repl
+.lein-repl-history
+.lsp
 .nrepl-port
-.cpcache
+.nyc_output
 .rebel_readline_history
-/vendor/**/target
-/cd.md
-node_modules
-/backend/target/
-/backend/resources/public/media
-/backend/resources/public/assets
+.repl
+/.clj-kondo/.cache
+/_dump
+/backend/-
+/backend/assets/
 /backend/dist/
 /backend/logs/
-/backend/-
-/frontend/npm-debug.log
-/frontend/target/
-/frontend/dist/
-/frontend/out/
-/frontend/.shadow-cljs
-/frontend/resources/public/*
-/exporter/target
-/exporter/.shadow-cljs
-/docker/images/bundle
-/.clj-kondo/.cache
+/backend/resources/public/assets
+/backend/resources/public/media
+/backend/target/
 /bundle*
-/media
+/cd.md
+/clj-profiler/
+/common/.shadow-cljs
+/common/coverage
+/common/target
 /deploy
+/docker/images/bundle*
+/exporter/.shadow-cljs
+/exporter/target
+/frontend/.shadow-cljs
+/frontend/cypress/videos/*/
+/frontend/dist/
+/frontend/npm-debug.log
+/frontend/out/
+/frontend/resources/fonts/experiments
+/frontend/resources/public/*
+/frontend/target/
+/media
+/telemetry/
+/vendor/**/target
+/vendor/svgclean/bundle*.js
 /web
-/_dump
-/vendor/svgclean/bundle*.js
+clj-profiler/
+figwheel_server.log
+node_modules

.gitpod.yml

@@ -0,0 +1,105 @@
+image:
+  file: docker/gitpod/Dockerfile
+
+ports:
+  # nginx
+  - port: 3449
+    onOpen: open-preview
+
+  # frontend nREPL
+  - port: 3447
+    onOpen: ignore
+    visibility: private
+
+  # frontend shadow server
+  - port: 3448
+    onOpen: ignore
+    visibility: private
+
+  # backend
+  - port: 6060
+    onOpen: ignore
+
+  # exporter shadow server
+  - port: 9630
+    onOpen: ignore
+    visibility: private
+
+  # exporter http server
+  - port: 6061
+    onOpen: ignore
+
+  # mailhog web interface
+  - port: 8025
+    onOpen: ignore
+
+  # mailhog postfix
+  - port: 1025
+    onOpen: ignore
+
+  # postgres
+  - port: 5432
+    onOpen: ignore
+
+  # redis
+  - port: 6379
+    onOpen: ignore
+
+  # openldap
+  - port: 389
+    onOpen: ignore
+
+tasks:
+  # https://github.com/gitpod-io/gitpod/issues/666#issuecomment-534347856
+  - name: gulp
+    command: >
+      cd $GITPOD_REPO_ROOT/frontend/;
+      yarn && gp sync-done 'frontend-yarn';
+      npx gulp --theme=${PENPOT_THEME} watch
+
+  - name: frontend shadow watch
+    command: >
+      cd $GITPOD_REPO_ROOT/frontend/;
+      gp sync-await 'frontend-yarn';
+      npx shadow-cljs watch main
+
+  - init: gp await-port 5432 && psql -f $GITPOD_REPO_ROOT/docker/gitpod/files/postgresql_init.sql
+    name: backend
+    command: >
+      cd $GITPOD_REPO_ROOT/backend/;
+      ./scripts/start-dev
+
+  - name: exporter shadow watch
+    command:
+      cd $GITPOD_REPO_ROOT/exporter/;
+      gp sync-await 'frontend-yarn';
+      yarn && npx shadow-cljs watch main
+
+  - name: exporter web server
+    command: >
+      cd $GITPOD_REPO_ROOT/exporter/;
+      ./scripts/wait-and-start.sh
+
+  - name: signed terminal
+    before: >
+      [[ ! -z ${GNUGPG}  ]] &&
+      cd ~ &&
+      rm -rf .gnupg &&
+      echo ${GNUGPG} | base64 -d | tar --no-same-owner -xzvf -
+    init: >
+      [[ ! -z ${GNUGPG_KEY}  ]] &&
+      git config --global commit.gpgsign true &&
+      git config --global user.signingkey ${GNUGPG_KEY}
+    command: cd $GITPOD_REPO_ROOT
+
+  - name: redis
+    command: redis-server
+
+  - before: go get github.com/mailhog/MailHog
+    name: mailhog
+    command: MailHog
+
+  - name: Nginx
+    command: >
+      nginx &&
+      multitail /var/log/nginx/access.log -I /var/log/nginx/error.log

CHANGES.md

@@ -0,0 +1,792 @@
+# CHANGELOG
+
+## 1.11.2-beta
+
+### :bug: Bugs fixed
+
+- Fix issue on handling empty content on boolean shapes
+- Fix race condition issue on component renaming
+- Handle EOF errors on writting streamed response
+- Handle EOF errors on websocket send/ping methods
+- Disable parallel upload of file media on import (causes too much
+  contention on the rlimit subsistem that does not works as expected
+  on high load).
+
+### :sparkles: New features
+
+- Add health check endpoint on API
+- Increase default max connection pool size to 60
+- Reduce resource usage of the error reporter.
+
+
+## 1.11.1-beta
+
+### :bug: Bugs fixed
+
+- Fix issue related to default http host config value.
+- Fix issue on rendering frames on firefox.
+
+### :arrow_up: Deps updates
+
+- Update nodejs version to 16.13.1 on docker images.
+
+
+## 1.11.0-beta
+
+### :boom: Breaking changes
+
+### :sparkles: New features
+
+- Add an option to hide artboards names on the viewport [Taiga #2034](https://tree.taiga.io/project/penpot/issue/2034)
+- Limit pasted object position to container boundaries [Taiga #2449](https://tree.taiga.io/project/penpot/us/2449)
+- Add new options for zoom widget in workspace and viewer mode [Taiga #896](https://tree.taiga.io/project/penpot/us/896)
+- Allow decimals on stroke width and positions [Taiga #2035](https://tree.taiga.io/project/penpot/issue/2035)
+- Ability to ignore background when exporting an artboard [Taiga #1395](https://tree.taiga.io/project/penpot/us/1395)
+- Show color hex or name on hover [Taiga #2413](https://tree.taiga.io/project/penpot/us/2413)
+- Add shortcut to create artboard from selected objects [Taiga #2412](https://tree.taiga.io/project/penpot/us/2412)
+- Add shortcut for opacity [Taiga #2442](https://tree.taiga.io/project/penpot/us/2442)
+- Setting fill automatically for new texts [Taiga #2441](https://tree.taiga.io/project/penpot/us/2441)
+- Add shortcut to move action [Github #1213](https://github.com/penpot/penpot/issues/1213)
+- Add alt as mod key to add stroke color from library menu [Taiga #2207](https://tree.taiga.io/project/penpot/us/2207)
+- Add detach in bulk option to context menu [Taiga #2210](https://tree.taiga.io/project/penpot/us/2210)
+- Add penpot look and feel to multiuser cursors [Taiga #1387](https://tree.taiga.io/project/penpot/us/1387)
+- Add actions to go to main component context menu option [Taiga #2053](https://tree.taiga.io/project/penpot/us/2053)
+- Add contrast between component select color and shape select color [Taiga #2121](https://tree.taiga.io/project/penpot/issue/2121)
+- Add animations in interactions [Taiga #2244](https://tree.taiga.io/project/penpot/us/2244)
+- Add performance improvements on .penpot file import process [Taiga #2497](https://tree.taiga.io/project/penpot/us/2497)
+- On team settings set color of members count to black [Taiga #2607](https://tree.taiga.io/project/penpot/us/2607)
+
+### :bug: Bugs fixed
+
+- Fix remove gradient if any when applying color from library [Taiga #2299](https://tree.taiga.io/project/penpot/issue/2299)
+- Fix Enter as key action to exit edit path [Taiga #2444](https://tree.taiga.io/project/penpot/issue/2444)
+- Fix add fill color from palette to groups and components [Taiga #2313](https://tree.taiga.io/project/penpot/issue/2313)
+- Fix default project name in all languages [Taiga #2280](https://tree.taiga.io/project/penpot/issue/2280)
+- Fix line-height and letter-spacing inputs to allow negative values [Taiga #2381](https://tree.taiga.io/project/penpot/issue/2381)
+- Fix typo in Handoff tooltip [Taiga #2428](https://tree.taiga.io/project/penpot/issue/2428)
+- Fix crash when pressing Shift+1 on empty file [#1435](https://github.com/penpot/penpot/issues/1435)
+- Fix masked group resize strange behavior [Taiga #2317](https://tree.taiga.io/project/penpot/issue/2317)
+- Fix problems when exporting all artboards [Taiga #2234](https://tree.taiga.io/project/penpot/issue/2234)
+- Fix problems with team management [#1353](https://github.com/penpot/penpot/issues/1353)
+- Fix problem when importing in shared libraries [#1362](https://github.com/penpot/penpot/issues/1362)
+- Fix problem with join nodes [#1422](https://github.com/penpot/penpot/issues/1422)
+- After team onboarding importing a file will import into the team drafts [Taiga #2408](https://tree.taiga.io/project/penpot/issue/2408)
+- Fix problem exporting shapes from handoff mode [Taiga #2386](https://tree.taiga.io/project/penpot/issue/2386)
+- Fix lock/hide elements in context menu when multiples shapes selected [Taiga #2340](https://tree.taiga.io/project/penpot/issue/2340)
+- Fix problem with booleans [Taiga #2356](https://tree.taiga.io/project/penpot/issue/2356)
+- Fix line-height/letter-spacing inputs behaviour [Taiga #2331](https://tree.taiga.io/project/penpot/issue/2331)
+- Fix dotted style in strokes [Taiga #2312](https://tree.taiga.io/project/penpot/issue/2312)
+- Fix problem when resizing texts inside groups [Taiga #2310](https://tree.taiga.io/project/penpot/issue/2310)
+- Fix problem with multiple exports [Taiga #2468](https://tree.taiga.io/project/penpot/issue/2468)
+- Allow import to continue from recoverable failures [#1412](https://github.com/penpot/penpot/issues/1412)
+- Improved behaviour on text options when not text is selected [Taiga #2390](https://tree.taiga.io/project/penpot/issue/2390)
+- Fix decimal numbers in export viewbox [Taiga #2290](https://tree.taiga.io/project/penpot/issue/2290)
+- Right click over artboard name to open its menu [Taiga #1679](https://tree.taiga.io/project/penpot/issue/1679)
+- Make the default session cookue use SameSite=Lax instead of Strict (causes some issues in latest versions of Chrome)
+- Fix "open in new tab" on dashboard [Taiga #2235](https://tree.taiga.io/project/penpot/issue/2355)
+- Changing pages while comments activated will not close the panel [#1350](https://github.com/penpot/penpot/issues/1350)
+- Fix navigate comments in right sidebar [Taiga #2163](https://tree.taiga.io/project/penpot/issue/2163)
+- Fix keep name of component equal to the shape name [Taiga #2341](https://tree.taiga.io/project/penpot/issue/2341)
+- Fix lossing changes when changing selection and an input was already changed [Taiga #2329](https://tree.taiga.io/project/penpot/issue/2329), [Taiga #2330](https://tree.taiga.io/project/penpot/issue/2330)
+- Fix blur input field when click on viewport [Taiga #2164](https://tree.taiga.io/project/penpot/issue/2164)
+- Fix default page id in workspace [Taiga #2205](https://tree.taiga.io/project/penpot/issue/2205)
+- Fix problem when importing a file with grids [Taiga #2314](https://tree.taiga.io/project/penpot/issue/2314)
+- Fix problem with imported svgs with filters [Taiga #2478](https://tree.taiga.io/project/penpot/issue/2478)
+- Fix issues when updating selrect in paths [Taiga #2366](https://tree.taiga.io/project/penpot/issue/2366)
+- Fix scroll jumps in handoff mode [Taiga #2383](https://tree.taiga.io/project/penpot/issue/2383)
+- Fix handoff text with opacity [Taiga #2384](https://tree.taiga.io/project/penpot/issue/2384)
+- Restored rules color [Taiga #2460](https://tree.taiga.io/project/penpot/issue/2460)
+- Fix thumbnail not taking frame blending mode [Taiga #2301](https://tree.taiga.io/project/penpot/issue/2301)
+- Fix import/export with SVG edge cases [Taiga #2389](https://tree.taiga.io/project/penpot/issue/2389)
+- Avoid modifying component when moving into a group [Taiga #2534](https://tree.taiga.io/project/penpot/issue/2534)
+- Show correctly group types label in handoff [Taiga #2482](https://tree.taiga.io/project/penpot/issue/2482)
+- Display view mode buttons always centered in viewer [#Taiga 2466](https://tree.taiga.io/project/penpot/issue/2466)
+- Fix default profile image generation issue [Taiga #2601](https://tree.taiga.io/project/penpot/issue/2601)
+- Fix edit blur attributes for multiselection [Taiga #2625](https://tree.taiga.io/project/penpot/issue/2625)
+- Fix auto hide header in viewer full screen [Taiga #2632](https://tree.taiga.io/project/penpot/issue/2632)
+- Fix zoom in/out after fit or fill [Taiga #2630](https://tree.taiga.io/project/penpot/issue/2630)
+- Normalize zoom levels in workspace and viewer [Taiga #2631](https://tree.taiga.io/project/penpot/issue/2631)
+- Avoid empty names in projects, files and pages [Taiga #2594](https://tree.taiga.io/project/penpot/issue/2594)
+- Fix "move to" menu when duplicated team or project names [Taiga #2655](https://tree.taiga.io/project/penpot/issue/2655)
+- Fix ungroup a component leaves an asterisk in layers [Taiga #2694](https://tree.taiga.io/project/penpot/issue/2694)
+
+### :arrow_up: Deps updates
+
+- Update devenv docker image dependencies.
+
+### :heart: Community contributions by (Thank you!)
+
+- Spelling fixes (by @jsoref) [#1340](https://github.com/penpot/penpot/pull/1340)
+- Explain folders in components (by @candideu) [Penpot-docs #42](https://github.com/penpot/penpot-docs/pull/42)
+- Readability improvements of user guide (by @PaulSchulz) [Penpot-docs #50](https://github.com/penpot/penpot-docs/pull/50)
+
+## 1.10.4-beta
+
+### :sparkles: Enhacements
+
+- Allow parametrice file snapshoting interval.
+
+### :bug: Bugs fixed
+
+- Fix issue on :mov-object change impl.
+- Minor fix on how file changes log is persisted.
+- Fix many issues on error reporting.
+
+## 1.10.3-beta
+
+### :sparkles: Enhacements
+
+- Make all logging asynchronous, this avoid some overhead on jetty threads at cost of logging latency.
+- Increase default session time to 15 days.
+
+### :bug: Bugs fixed
+
+- Fix unexpected exception on saving pages with default grids [#2409](https://tree.taiga.io/project/penpot/issue/2409)
+- Fix react warnings on setting size 1 on row and column grids.
+- Fix minor issues on ZMQ logging listener (used in error reporting service)
+- Remove "ALPHA" from the code.
+- Fix value and nil handling on numeric-input component. This fixes many issues related to typography, components, etc. renaming.
+- Fix NPE on email complains processing.
+- Fix white page after leaving a team.
+- Fix missing leave team button outside members page.
+
+### :arrow_up: Deps updates
+
+- Update log4j2 dependency.
+
+## 1.10.2-beta
+
+### :bug: Bugs fixed
+
+- Fix corner case issues with media file uploads.
+- Fix issue with default page grids validation.
+- Fix issue related to some raceconditions on workspace navigation events.
+
+### :arrow_up: Deps updates
+
+- Update log4j2 dependency.
+
+## 1.10.1-beta
+
+### :bug: Bugs fixed
+
+- Fix problems with team management [#1353](https://github.com/penpot/penpot/issues/1353)
+
+## 1.10.0-beta
+
+### :boom: Breaking changes
+
+- The initial project / data mechanism (not documented) has been
+  disabled. Is the mechanism used for creating initial project on user
+  signup. With the new onboarding approach, this subsystem is no
+  longer needed and is disabled.
+
+### :sparkles: New features
+
+- Allow ungroup groups in bulk [Taiga #2211](https://tree.taiga.io/project/penpot/us/2211)
+- Enhance corner radius behavior [Taiga #2190](https://tree.taiga.io/project/penpot/issue/2190)
+- Allow preserve scroll position in interactions [Taiga #2250](https://tree.taiga.io/project/penpot/us/2250)
+- Add new onboarding modals.
+
+### :bug: Bugs fixed
+
+- Fix problem with exporting before the document is saved [Taiga #2189](https://tree.taiga.io/project/penpot/issue/2189)
+- Fix undo stacking when changing color from color-picker [Taiga #2191](https://tree.taiga.io/project/penpot/issue/2191)
+- Fix pages dropdown in viewer [Taiga #2087](https://tree.taiga.io/project/penpot/issue/2087)
+- Fix problem when exporting texts with gradients or opacity [Taiga #2200](https://tree.taiga.io/project/penpot/issue/2200)
+- Fix problem with view mode comments [Taiga #2226](https://tree.taiga.io/project/penpot/issue/2226)
+- Disallow to create a component when already has one [Taiga #2237](https://tree.taiga.io/project/penpot/issue/2237)
+- Add ellipsis in long labels for input fields [Taiga #2224](https://tree.taiga.io/project/penpot/issue/2224)
+- Fix problem with text rendering on export [Taiga #2223](https://tree.taiga.io/project/penpot/issue/2223)
+- Fix problem when flattening booleans losing styles [Taiga #2217](https://tree.taiga.io/project/penpot/issue/2217)
+- Add shortcuts to boolean icons popups [Taiga #2220](https://tree.taiga.io/project/penpot/issue/2220)
+- Fix a worker error when transforming a rectangle into path
+- Fix max/min values for opacity fields [Taiga #2183](https://tree.taiga.io/project/penpot/issue/2183)
+- Fix viewer comment position when zoom applied [Taiga #2240](https://tree.taiga.io/project/penpot/issue/2240)
+- Remove change style on hover for options [Taiga #2172](https://tree.taiga.io/project/penpot/issue/2172)
+- Fix problem in viewer with dropdowns when comments active [#1303](https://github.com/penpot/penpot/issues/1303)
+- Add placeholder to create shareable link
+- Fix project files count not refreshing correctly after import [Taiga #2216](https://tree.taiga.io/project/penpot/issue/2216)
+- Remove button after import process finish [Taiga #2215](https://tree.taiga.io/project/penpot/issue/2215)
+- Fix problem with styles in the viewer [Taiga #2467](https://tree.taiga.io/project/penpot/issue/2467)
+- Fix default state in viewer [Taiga #2465](https://tree.taiga.io/project/penpot/issue/2465)
+- Fix division by zero in bool operation [Taiga #2349](https://tree.taiga.io/project/penpot/issue/2349)
+
+### :heart: Community contributions by (Thank you!)
+
+- To the translation community for the hard work on making penpot
+  available on so many languages.
+- Guide to integrate with Azure Directory (by @skrzyneckik) [Penpot-docs #33](https://github.com/penpot/penpot-docs/pull/33)
+- Improve libraries section readability (by @PaulSchulz) [Penpot-docs #39](https://github.com/penpot/penpot-docs/pull/39)
+
+## 1.9.0-alpha
+
+### :boom: Breaking changes
+
+- Some stroke-caps can change behaviour.
+- Text display bug fix could potentially make some texts jump a line.
+
+### :sparkles: New features
+
+- Add boolean shapes: intersections, unions, difference and exclusions[Taiga #748](https://tree.taiga.io/project/penpot/us/748)
+- Add advanced prototyping [Taiga #244](https://tree.taiga.io/project/penpot/us/244)
+- Add multiple flows [Taiga #2091](https://tree.taiga.io/project/penpot/us/2091)
+- Change order of the teams menu so it's in the joined time order.
+
+### :bug: Bugs fixed
+
+- Enhance duplicating prototype connections behaviour [Taiga #2093](https://tree.taiga.io/project/penpot/us/2093)
+- Ignore constraints in horizontal or vertical flip [Taiga #2038](https://tree.taiga.io/project/penpot/issue/2038)
+- Fix color and typographies refs lost when duplicated file [Taiga #2165](https://tree.taiga.io/project/penpot/issue/2165)
+- Fix problem with overflow dropdown on stroke-cap [#1216](https://github.com/penpot/penpot/issues/1216)
+- Fix menu context for single element nested in components [#1186](https://github.com/penpot/penpot/issues/1186)
+- Fix error screen when operations over comments fail [#1219](https://github.com/penpot/penpot/issues/1219)
+- Fix undo problem when changing typography/color from library [#1230](https://github.com/penpot/penpot/issues/1230)
+- Fix problem with text margin while rendering [#1231](https://github.com/penpot/penpot/issues/1231)
+- Fix problem with masked texts on exporting [Taiga #2116](https://tree.taiga.io/project/penpot/issue/2116)
+- Fix text editor enter behaviour with centered texts [Taiga #2126](https://tree.taiga.io/project/penpot/issue/2126)
+- Fix residual stroke on imported svg [Taiga #2125](https://tree.taiga.io/project/penpot/issue/2125)
+- Add links for terms of service and privacy policy in register checkbox [Taiga #2020](https://tree.taiga.io/project/penpot/issue/2020)
+- Allow three character hex and web colors in color picker hex input [#1184](https://github.com/penpot/penpot/issues/1184)
+- Allow lowercase search for fonts [#1180](https://github.com/penpot/penpot/issues/1180)
+- Fix group renaming problem [Taiga #1969](https://tree.taiga.io/project/penpot/issue/1969)
+- Fix export group with shadows on children [Taiga #2036](https://tree.taiga.io/project/penpot/issue/2036)
+- Fix zoom context menu in viewer [Taiga #2041](https://tree.taiga.io/project/penpot/issue/2041)
+- Fix stroke caps adjustments in relation with stroke size [Taiga #2123](https://tree.taiga.io/project/penpot/issue/2123)
+- Fix problem duplicating paths [Taiga #2147](https://tree.taiga.io/project/penpot/issue/2147)
+- Fix problem inheriting attributes from SVG root when importing [Taiga #2124](https://tree.taiga.io/project/penpot/issue/2124)
+- Fix problem with lines and inside/outside stroke [Taiga #2146](https://tree.taiga.io/project/penpot/issue/2146)
+- Add stroke width in selection calculation [Taiga #2146](https://tree.taiga.io/project/penpot/issue/2146)
+- Fix shift+wheel to horizontal scrolling in MacOS [#1217](https://github.com/penpot/penpot/issues/1217)
+- Fix path stroke is not working properly with high thickness [Taiga #2154](https://tree.taiga.io/project/penpot/issue/2154)
+- Fix bug with transformation operations [Taiga #2155](https://tree.taiga.io/project/penpot/issue/2155)
+- Fix bug in firefox when a text box is inside a mask [Taiga #2152](https://tree.taiga.io/project/penpot/issue/2152)
+- Fix problem with stroke inside/outside [Taiga #2186](https://tree.taiga.io/project/penpot/issue/2186)
+- Fix masks export area [Taiga #2189](https://tree.taiga.io/project/penpot/issue/2189)
+- Fix paste in place in artboards [Taiga #2188](https://tree.taiga.io/project/penpot/issue/2188)
+- Fix font size input stuck on selection change [Taiga #2184](https://tree.taiga.io/project/penpot/issue/2184)
+- Fix stroke cut on shapes export [Taiga #2171](https://tree.taiga.io/project/penpot/issue/2171)
+- Fix no color when boolean with an SVG [Taiga #2193](https://tree.taiga.io/project/penpot/issue/2193)
+- Fix unlink color styles at strokes [Taiga #2206](https://tree.taiga.io/project/penpot/issue/2206)
+
+### :arrow_up: Deps updates
+
+### :heart: Community contributions by (Thank you!)
+
+- To the translation community for the hard work on making penpot
+  available on so many languages.
+
+## 1.8.4-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem importing components [Taiga #2151](https://tree.taiga.io/project/penpot/issue/2151)
+
+## 1.8.3-alpha
+
+### :sparkles: New features
+
+- Adds progress report to importing process.
+
+## 1.8.2-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with masking images in viewer [#1238](https://github.com/penpot/penpot/issues/1238)
+
+## 1.8.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix project renaming issue (and some other related to the same underlying bug)
+- Fix internal exception on audit log persistence layer.
+- Set proper environment variable on docker images for chrome executable.
+- Fix internal metrics on websocket connections.
+
+## 1.8.0-alpha
+
+### :boom: Breaking changes
+
+- This release includes a new approach for handling share links, and
+  this feature is incompatible with the previous one. This means that
+  all the public share links generated previously will stop working.
+
+### :sparkles: New features
+
+- Add tooltips to color picker tabs [Taiga #1814](https://tree.taiga.io/project/penpot/us/1814)
+- Add styling to the end point of any open paths [Taiga #1107](https://tree.taiga.io/project/penpot/us/1107)
+- Allow to zoom with ctrl + middle button [Taiga #1428](https://tree.taiga.io/project/penpot/us/1428)
+- Auto placement of duplicated objects [Taiga #1386](https://tree.taiga.io/project/penpot/us/1386)
+- Enable penpot SVG metadata only when exporting complete files [Taiga #1914](https://tree.taiga.io/project/penpot/us/1914?milestone=295883)
+- Export to PDF all artboards of one page [Taiga #1895](https://tree.taiga.io/project/penpot/us/1895)
+- Go to a undo step clicking on a history element of the list [Taiga #1374](https://tree.taiga.io/project/penpot/us/1374)
+- Increment font size by 10 with shift+arrows [1047](https://github.com/penpot/penpot/issues/1047)
+- New shortcut to detach components Ctrl+Shift+K [Taiga #1799](https://tree.taiga.io/project/penpot/us/1799)
+- Set email inputs to type "email", to aid keyboard entry [Taiga #1921](https://tree.taiga.io/project/penpot/issue/1921)
+- Use shift+move to move element orthogonally [#823](https://github.com/penpot/penpot/issues/823)
+- Use space + mouse drag to pan, instead of only space [Taiga #1800](https://tree.taiga.io/project/penpot/us/1800)
+- Allow navigate through pages on the viewer [Taiga #1550](https://tree.taiga.io/project/penpot/us/1550)
+- Allow create share links with specific pages [Taiga #1844](https://tree.taiga.io/project/penpot/us/1844)
+
+### :bug: Bugs fixed
+
+- Prevent adding numeric suffix to layer names when not needed [Taiga #1929](https://tree.taiga.io/project/penpot/us/1929)
+- Prevent deleting or moving the drafts project [Taiga #1935](https://tree.taiga.io/project/penpot/issue/1935)
+- Fix problem with zoom and selection [Taiga #1919](https://tree.taiga.io/project/penpot/issue/1919)
+- Fix problem with borders on shape export [#1092](https://github.com/penpot/penpot/issues/1092)
+- Fix thumbnail cropping issue [Taiga #1964](https://tree.taiga.io/project/penpot/issue/1964)
+- Fix repeated fetch on file selection [Taiga #1933](https://tree.taiga.io/project/penpot/issue/1933)
+- Fix rename typography on text options [Taiga #1963](https://tree.taiga.io/project/penpot/issue/1963)
+- Fix problems with order in groups [Taiga #1960](https://tree.taiga.io/project/penpot/issue/1960)
+- Fix SVG components preview [#1134](https://github.com/penpot/penpot/issues/1134)
+- Fix group renaming problem [Taiga #1969](https://tree.taiga.io/project/penpot/issue/1969)
+- Fix problem with import broken images links [#1197](https://github.com/penpot/penpot/issues/1197)
+- Fix problem while moving imported SVG's [#1199](https://github.com/penpot/penpot/issues/1199)
+
+### :arrow_up: Deps updates
+
+### :boom: Breaking changes
+
+### :heart: Community contributions by (Thank you!)
+
+- eduayme [#1129](https://github.com/penpot/penpot/pull/1129)
+
+## 1.7.4-alpha
+
+### :bug: Bugs fixed
+
+- Fix demo user creation (self-hosted only)
+- Add better ldap response validation and reporting (self-hosted only)
+
+## 1.7.3-alpha
+
+### :bug: Bugs fixed
+
+- Fix font uploading issue on Windows.
+
+## 1.7.2-alpha
+
+### :sparkles: New features
+
+- Add many improvements to text tool.
+
+### :bug: Bugs fixed
+
+- Add scroll bar to Teams menu [Taiga #1894](https://tree.taiga.io/project/penpot/issue/1894)
+- Fix repeated names when duplicating artboards or groups [Taiga #1892](https://tree.taiga.io/project/penpot/issue/1892)
+- Fix properly messages lifecycle on navigate.
+- Fix handling repeated names on duplicate object trees.
+- Fix group naming on group creation.
+- Fix some issues in svg transformation.
+
+### :arrow_up: Deps updates
+
+- Update frontend build tooling.
+
+### :heart: Community contributions by (Thank you!)
+
+- soultipsy [#1100](https://github.com/penpot/penpot/pull/1100)
+
+## 1.7.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix issue related to the GC and images in path shapes.
+- Fix issue on the shape order on some undo operations.
+- Fix issue on undo page deletion.
+- Fix some issues related to constraints.
+
+## 1.7.0-alpha
+
+### :sparkles: New features
+
+- Allow nested asset groups [Taiga #1716](https://tree.taiga.io/project/penpot/us/1716)
+- Allow to ungroup assets [Taiga #1719](https://tree.taiga.io/project/penpot/us/1719)
+- Allow to rename assets groups [Taiga #1721](https://tree.taiga.io/project/penpot/us/1721)
+- Component constraints (left, right, left and right, center, scale...) [Taiga #1125](https://tree.taiga.io/project/penpot/us/1125)
+- Export elements to PDF [Taiga #519](https://tree.taiga.io/project/penpot/us/519)
+- Memorize collapse state of assets in panel [Taiga #1718](https://tree.taiga.io/project/penpot/us/1718)
+- Headers button sets and menus review [Taiga #1663](https://tree.taiga.io/project/penpot/us/1663)
+- Preserve components if possible, when pasted into a different file [Taiga #1063](https://tree.taiga.io/project/penpot/issue/1063)
+- Add the ability to offload file data to a cheaper storage when file becomes inactive.
+- Import/Export Penpot files from dashboard.
+- Double click won't make a shape a path until you change a node [Taiga #1796](https://tree.taiga.io/project/penpot/us/1796)
+- Incremental area selection [#779](https://github.com/penpot/penpot/discussions/779)
+
+### :bug: Bugs fixed
+
+- Process numeric input changes only if the value actually changed.
+- Remove unnecessary redirect from history when user goes to workspace from dashboard [Taiga #1820](https://tree.taiga.io/project/penpot/issue/1820)
+- Detach shapes from deleted assets [Taiga #1850](https://tree.taiga.io/project/penpot/issue/1850)
+- Fix tooltip position on view application [Taiga #1819](https://tree.taiga.io/project/penpot/issue/1819)
+- Fix dashboard navigation on moving file to other team [Taiga #1817](https://tree.taiga.io/project/penpot/issue/1817)
+- Fix workspace header presence styles and invalid link [Taiga #1813](https://tree.taiga.io/project/penpot/issue/1813)
+- Fix color-input wrong behavior (on workspace page color) [Taiga #1795](https://tree.taiga.io/project/penpot/issue/1795)
+- Fix file contextual menu in shared libraries at dashboard [Taiga #1865](https://tree.taiga.io/project/penpot/issue/1865)
+- Fix problem with color picker and fonts [#1049](https://github.com/penpot/penpot/issues/1049)
+- Fix negative values in blur [Taiga #1815](https://tree.taiga.io/project/penpot/issue/1815)
+- Fix problem when editing color in group [Taiga #1816](https://tree.taiga.io/project/penpot/issue/1816)
+- Fix resize/rotate with mouse buttons different than left [#1060](https://github.com/penpot/penpot/issues/1060)
+- Fix header partially visible on fullscreen viewer mode [Taiga #1875](https://tree.taiga.io/project/penpot/issue/1875)
+- Fix dynamic alignment enabled with hidden objects [#1063](https://github.com/penpot/penpot/issues/1063)
+
+## 1.6.5-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with paths editing after flip [#1040](https://github.com/penpot/penpot/issues/1040)
+
+## 1.6.4-alpha
+
+### :sparkles: Minor improvements
+
+- Decrease default bulk buffers on storage tasks.
+- Reduce file_change preserve interval to 24h.
+
+### :bug: Bugs fixed
+
+- Don't allow rename drafts project.
+- Fix custom font deletion task.
+- Fix custom font rendering on exporting shapes.
+- Fix font loading on viewer app.
+- Fix problem when moving files with drag & drop.
+- Fix unexpected exception on searching without term.
+- Properly handle nil values on `update-shapes` function.
+- Replace frame term usage by artboard on viewer app.
+
+## 1.6.3-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with merge and join nodes [#990](https://github.com/penpot/penpot/issues/990)
+- Fix problem with empty path editing.
+- Fix problem with create component.
+- Fix problem with move-objects.
+- Fix problem with merge and join nodes.
+
+## 1.6.2-alpha
+
+### :bug: Bugs fixed
+
+- Add better auth module logging.
+- Add missing `email` scope to OIDC backend.
+- Add missing cause prop on error loging.
+- Fix empty font-family handling on custom fonts page.
+- Fix incorrect unicode code points handling on draft-to-penpot conversion.
+- Fix some problems with paths.
+- Fix unexpected exception on duplicate project.
+- Fix unexpected exception when user leaves typography name empty.
+- Improve error report on uploading invalid image to library.
+- Minor fix on previous commit.
+- Minor improvements on svg uploading on libraries.
+
+## 1.6.1-alpha
+
+### :bug: Bugs fixed
+
+- Add safety check on reg-objects change impl.
+- Fix custom fonts embedding issue.
+- Fix dashboard ordering issue.
+- Fix problem when creating a component with empty data.
+- Fix problem with moving shapes into frames.
+- Fix problems with mov-objects.
+- Fix unexpected exception related to rounding integers.
+- Fix wrong type usage on libraries changes.
+- Improve editor lifecycle management.
+- Make the navigation async by default.
+
+## 1.6.0-alpha
+
+### :sparkles: New features
+
+- Add improved workspace font selector [Taiga US #292](https://tree.taiga.io/project/penpot/us/292)
+- Add option to interactively scale text [Taiga #1527](https://tree.taiga.io/project/penpot/us/1527)
+- Add performance improvements on dashboard data loading.
+- Add performance improvements to indexes handling on workspace.
+- Add the ability to upload/use custom fonts (and automatically generate all needed webfonts) [Taiga US #292](https://tree.taiga.io/project/penpot/us/292)
+- Transform shapes to path on double click
+- Translate automatic names of new files and projects.
+- Use shift instead of ctrl/cmd to keep aspect ratio [Taiga 1697](https://tree.taiga.io/project/penpot/issue/1697)
+- New translations: Portuguese (Brazil) and Romanias.
+
+### :bug: Bugs fixed
+
+- Remove interactions when the destination artboard is deleted [Taiga #1656](https://tree.taiga.io/project/penpot/issue/1656)
+- Fix problem with fonts that ends with numbers [#940](https://github.com/penpot/penpot/issues/940)
+- Fix problem with imported SVG on editing paths [#971](https://github.com/penpot/penpot/issues/971)
+- Fix problem with color picker positioning
+- Fix order on color palette [#961](https://github.com/penpot/penpot/issues/961)
+- Fix issue when group creation leaves an empty group [#1724](https://tree.taiga.io/project/penpot/issue/1724)
+- Fix problem with :multiple for colors and typographies [#1668](https://tree.taiga.io/project/penpot/issue/1668)
+- Fix problem with locked shapes when change parents [#974](https://github.com/penpot/penpot/issues/974)
+- Fix problem with new nodes in paths [#978](https://github.com/penpot/penpot/issues/978)
+
+### :arrow_up: Deps updates
+
+- Update exporter dependencies (puppeteer), that fixes some unexpected exceptions.
+- Update string manipulation library.
+
+### :boom: Breaking changes
+
+- The OIDC setting `PENPOT_OIDC_SCOPES` has changed the default semantics. Before this
+  configuration added scopes to the default set. Now it replaces it, so use with care, because
+  penpot requires at least `name` and `email` props found on the user info object.
+
+### :heart: Community contributions by (Thank you!)
+
+- Translations: Portuguese (Brazil) and Romanias.
+
+## 1.5.4-alpha
+
+### :bug: Bugs fixed
+
+- Fix issues on group rendering.
+- Fix problem with text editing auto-height [Taiga #1683](https://tree.taiga.io/project/penpot/issue/1683)
+
+## 1.5.3-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem undo/redo.
+
+## 1.5.2-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with `close-path` command [#917](https://github.com/penpot/penpot/issues/917)
+- Fix wrong query for obtain the profile default project-id
+- Fix problems with empty paths and shortcuts [#923](https://github.com/penpot/penpot/issues/923)
+
+## 1.5.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix issue with bitmap image clipboard.
+- Fix issue when removing all path points.
+- Increase default team invitation token expiration to 48h.
+- Fix wrong error message when an expired token is used.
+
+## 1.5.0-alpha
+
+### :sparkles: New features
+
+- Add integration with gitpod.io (an online IDE) [#807](https://github.com/penpot/penpot/pull/807)
+- Allow basic math operations in inputs [Taiga 1383](https://tree.taiga.io/project/penpot/us/1383)
+- Autocomplete color names in hex inputs [Taiga 1596](https://tree.taiga.io/project/penpot/us/1596)
+- Allow to group assets (components and graphics) [Taiga #1289](https://tree.taiga.io/project/penpot/us/1289)
+- Change icon of pinned projects [Taiga 1298](https://tree.taiga.io/project/penpot/us/1298)
+- Internal: refactor of http client, replace internal xhr usage with more modern Fetch API.
+- New features for paths: snap points on edition, add/remove nodes, merge/join/split nodes. [Taiga #907](https://tree.taiga.io/project/penpot/us/907)
+- Add OpenID-Connect support.
+- Reimplement social auth providers on top of the generic openid impl.
+
+### :bug: Bugs fixed
+
+- Fix problem with pan and space [#811](https://github.com/penpot/penpot/issues/811)
+- Fix issue when parsing exponential numbers in paths
+- Remove legacy system user and team [#843](https://github.com/penpot/penpot/issues/843)
+- Fix ordering of copy pasted objects [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1617)
+- Fix problems with blending modes [#837](https://github.com/penpot/penpot/issues/837)
+- Fix problem with zoom an selection rect [#845](https://github.com/penpot/penpot/issues/845)
+- Fix problem displaying team statistics [#859](https://github.com/penpot/penpot/issues/859)
+- Fix problems with text editor selection [Taiga #1546](https://tree.taiga.io/project/penpot/issue/1546)
+- Fix problem when opening the context menu in dashboard at the bottom [#856](https://github.com/penpot/penpot/issues/856)
+- Fix problem when clicking an interactive group in view mode [#863](https://github.com/penpot/penpot/issues/863)
+- Fix visibility of pages in sitemap when changing page [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1618)
+- Fix visual problem with group invite [Taiga #1290](https://tree.taiga.io/project/penpot/issue/1290)
+- Fix issues with promote owner panel [Taiga #763](https://tree.taiga.io/project/penpot/issue/763)
+- Allow use library colors when defining gradients [Taiga #1614](https://tree.taiga.io/project/penpot/issue/1614)
+- Fix group selrect not updating after alignment [#895](https://github.com/penpot/penpot/issues/895)
+
+### :arrow_up: Deps updates
+
+### :boom: Breaking changes
+
+- Translations refactor: now penpot uses gettext instead of a custom
+  JSON, and each locale has its own separated file. All translations
+  should be contributed via the weblate.org service.
+
+### :heart: Community contributions by (Thank you!)
+
+- madmath03 (by [Monogramm](https://github.com/Monogramm)) [#807](https://github.com/penpot/penpot/pull/807)
+- zzkt [#814](https://github.com/penpot/penpot/pull/814)
+
+## 1.4.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix typography unlinking.
+- Fix incorrect measures on shapes outside artboard.
+- Fix issues on svg parsing related to numbers with exponents.
+- Fix some race conditions on removing shape from workspace.
+- Fix incorrect state management of user lang selection.
+- Fix email validation usability issue on team invitation lightbox.
+
+## 1.4.0-alpha
+
+### :sparkles: New features
+
+- Add blob-encoding v3 (uses ZSTD+transit) [#738](https://github.com/penpot/penpot/pull/738)
+- Add http caching layer on top of Query RPC.
+- Add layer opacity and blend mode to shapes [Taiga #937](https://tree.taiga.io/project/penpot/us/937)
+- Add more chinese translations [#726](https://github.com/penpot/penpot/pull/726)
+- Add native support for text-direction (RTL, LTR & auto)
+- Add several enhancements in shape selection [Taiga #1195](https://tree.taiga.io/project/penpot/us/1195)
+- Add thumbnail in memory caching mechanism.
+- Add turkish translation strings [#759](https://github.com/penpot/penpot/pull/759), [#794](https://github.com/penpot/penpot/pull/794)
+- Duplicate and move files and projects [Taiga #267](https://tree.taiga.io/project/penpot/us/267)
+- Hide viewer navbar on fullscreen [Taiga 1375](https://tree.taiga.io/project/penpot/us/1375)
+- Import SVG will create Penpot's shapes [Taiga #1006](https://tree.taiga.io/project/penpot/us/1066)
+- Improve french translations [#731](https://github.com/penpot/penpot/pull/731)
+- Reimplement workspace presence (remove database state)
+- Remember last visited team when you re-enter the application [Taiga #1376](https://tree.taiga.io/project/penpot/us/1376)
+- Rename artboard with double click on the title [Taiga #1392](https://tree.taiga.io/project/penpot/us/1392)
+- Replace Slate-Editor with DraftJS [Taiga #1346](https://tree.taiga.io/project/penpot/us/1346)
+- Set proper page title [Taiga #1377](https://tree.taiga.io/project/penpot/us/1377)
+
+### :bug: Bugs fixed
+
+- Disable buttons in view mode for users without permissions [Taiga #1328](https://tree.taiga.io/project/penpot/issue/1328)
+- Fix broken profile and profile options form.
+- Fix calculate size of some animated gifs [Taiga #1487](https://tree.taiga.io/project/penpot/issue/1487)
+- Fix error with the "Navigate to" button on prototypes [Taiga #1268](https://tree.taiga.io/project/penpot/issue/1268)
+- Fix issue when undo after changing the artboard of a shape [Taiga #1304](https://tree.taiga.io/project/penpot/issue/1304)
+- Fix issue with Alt key in distance measurement [#672](https://github.com/penpot/penpot/issues/672)
+- Fix issue with blending modes in masks [Taiga #1476](https://tree.taiga.io/project/penpot/issue/1476)
+- Fix issue with blocked shapes [Taiga #1480](https://tree.taiga.io/project/penpot/issue/1480)
+- Fix issue with comments styles on dashboard [Taiga #1405](https://tree.taiga.io/project/penpot/issue/1405)
+- Fix issue with default square grid [Taiga #1344](https://tree.taiga.io/project/penpot/issue/1344)
+- Fix issue with enter key shortcut [#775](https://github.com/penpot/penpot/issues/775)
+- Fix issue with enter to edit paths [Taiga #1481](https://tree.taiga.io/project/penpot/issue/1481)
+- Fix issue with mask and flip [#715](https://github.com/penpot/penpot/issues/715)
+- Fix issue with masks interactions outside bounds [#718](https://github.com/penpot/penpot/issues/718)
+- Fix issue with middle mouse button press moving the canvas when not moving mouse [#717](https://github.com/penpot/penpot/issues/717)
+- Fix issue with resolved comments [Taiga #1406](https://tree.taiga.io/project/penpot/issue/1406)
+- Fix issue with rotated blur [Taiga #1370](https://tree.taiga.io/project/penpot/issue/1370)
+- Fix issue with rotation degree input [#741](https://github.com/penpot/penpot/issues/741)
+- Fix issue with system shortcuts and application [#737](https://github.com/penpot/penpot/issues/737)
+- Fix issue with team management in dashboard [Taiga #1475](https://tree.taiga.io/project/penpot/issue/1475)
+- Fix issue with typographies panel cannot be collapsed [#707](https://github.com/penpot/penpot/issues/707)
+- Fix text selection in comments [#745](https://github.com/penpot/penpot/issues/745)
+- Update Work-Sans font [#744](https://github.com/penpot/penpot/issues/744)
+- Fix issue with recent files not showing [Taiga #1493](https://tree.taiga.io/project/penpot/issue/1493)
+- Fix issue when promoting to owner [Taiga #1494](https://tree.taiga.io/project/penpot/issue/1494)
+- Fix cannot click on blocked elements in viewer [Taiga #1430](https://tree.taiga.io/project/penpot/issue/1430)
+- Fix SVG not showing properties at code [Taiga #1437](https://tree.taiga.io/project/penpot/issue/1437)
+- Fix shadows when exporting groups [Taiga #1495](https://tree.taiga.io/project/penpot/issue/1495)
+- Fix drag-select when renaming layer text [Taiga #1307](https://tree.taiga.io/project/penpot/issue/1307)
+- Fix layout problem for editable selects [Taiga #1488](https://tree.taiga.io/project/penpot/issue/1488)
+- Fix artboard title wasn't move when resizing [Taiga #1479](https://tree.taiga.io/project/penpot/issue/1479)
+- Fix titles in viewer thumbnails too long [Taiga #1474](https://tree.taiga.io/project/penpot/issue/1474)
+- Fix when right click on a selected text shows artboard contextual menu [Taiga #1226](https://tree.taiga.io/project/penpot/issue/1226)
+
+### :boom: Breaking changes
+
+- The LDAP configuration variables interpolation starts using `:`
+  (example `:username`) instead of `$`. The main reason is avoid
+  unnecessary conflict with bash interpolation.
+
+### :arrow_up: Deps updates
+
+- Update backend to JDK16.
+- Update exporter nodejs to v14.16.0
+
+### :heart: Community contributions by (Thank you!)
+
+- iblueer [#726](https://github.com/penpot/penpot/pull/726)
+- gizembln [#759](https://github.com/penpot/penpot/pull/759)
+- girafic [#748](https://github.com/penpot/penpot/pull/748)
+- mbrksntrk [#794](https://github.com/penpot/penpot/pull/794)
+
+## 1.3.0-alpha
+
+### :sparkles: New features
+
+- Add emailcatcher and ldap test containers to devenv. [#506](https://github.com/penpot/penpot/pull/506)
+- Add major refactor of internal pubsub/redis code; improves scalability and performance [#640](https://github.com/penpot/penpot/pull/640)
+- Add more chinese translations [#687](https://github.com/penpot/penpot/pull/687)
+- Add more presets for artboard [#654](https://github.com/penpot/penpot/pull/654)
+- Add optional loki integration [#645](https://github.com/penpot/penpot/pull/645)
+- Add proper http session lifecycle handling.
+- Allow to set border radius of each rect corner individually
+- Bounce & Complaint handling [#635](https://github.com/penpot/penpot/pull/635)
+- Disable groups interactions when holding "Ctrl" key (deep selection)
+- New action in context menu to "edit" some shapes (bound to key "Enter")
+
+### :bug: Bugs fixed
+
+- Add more improvements to french translation strings [#591](https://github.com/penpot/penpot/pull/591)
+- Add some missing database indexes (mainly improves performance on large databases on file-update rpc method, and some background tasks)
+- Disables filters in masking elements (issue with Firefox rendering)
+- Drawing tool will have priority over resize/rotate handlers [Taiga #1225](https://tree.taiga.io/project/penpot/issue/1225)
+- Fix broken bounding box on editing paths [Taiga #1254](https://tree.taiga.io/project/penpot/issue/1254)
+- Fix corner cases on invitation/signup flows.
+- Fix errors on onboarding file [Taiga #1287](https://tree.taiga.io/project/penpot/issue/1287)
+- Fix infinite recursion on logout.
+- Fix issues with frame selection [Taiga #1300](https://tree.taiga.io/project/penpot/issue/1300), [Taiga #1255](https://tree.taiga.io/project/penpot/issue/1255)
+- Fix local fonts error [#691](https://github.com/penpot/penpot/issues/691)
+- Fix issue width handoff code generation [Taiga #1204](https://tree.taiga.io/project/penpot/issue/1204)
+- Fix issue with indices refreshing on page changes [#646](https://github.com/penpot/penpot/issues/646)
+- Have language change notification written in the new language [Taiga #1205](https://tree.taiga.io/project/penpot/issue/1205)
+- Hide register screen when registration is disabled [#598](https://github.com/penpot/penpot/issues/598)
+- Properly handle errors on github, gitlab and ldap auth backends.
+- Properly mark profile auth backend (on first register/ auth with 3rd party auth provider)
+- Refactor LDAP auth backend.
+
+### :heart: Community contributions by (Thank you!)
+
+- girafic [#538](https://github.com/penpot/penpot/pull/654)
+- arkhi [#591](https://github.com/penpot/penpot/pull/591)
+
+## 1.2.0-alpha
+
+### :sparkles: New features
+
+- Add horizontal/vertical flip
+- Add images lock proportions by default [#541](https://github.com/penpot/penpot/discussions/541), [#609](https://github.com/penpot/penpot/issues/609)
+- Add new blob storage format (Zstd+nippy)
+- Add user feedback form
+- Improve French translations
+- Improve component testing
+- Increase default deletion delay to 7 days
+- Show a pixel grid when zoom greater than 800% [#519](https://github.com/penpot/penpot/discussions/519)
+- Fix behavior of select all command when there are objects outside frames [Taiga #1209](https://tree.taiga.io/project/penpot/issue/1209)
+
+### :bug: Bugs fixed
+
+- Fix 404 when access shared link [#615](https://github.com/penpot/penpot/issues/615)
+- Fix 500 when requestion password reset
+- Fix issue when transforming path shapes [Taiga #1170](https://tree.taiga.io/project/penpot/issue/1170)
+- Fix apply a color to a text selection from color palette was not working [Taiga #1189](https://tree.taiga.io/project/penpot/issue/1189)
+- Fix issues when moving shapes outside groups [Taiga #1138](https://tree.taiga.io/project/penpot/issue/1138)
+- Fix ldap function called on login click
+- Fix logo icon in viewer should go to dashboard [Taiga #1149](https://tree.taiga.io/project/penpot/issue/1149)
+- Fix ordering when restoring deleted shapes in sync [Taiga #1163](https://tree.taiga.io/project/penpot/issue/1163)
+- Fix issue when editing text immediately after creating [Taiga #1207](https://tree.taiga.io/project/penpot/issue/1207)
+- Fix issue when pasting URL's copied from the browser url bar [Taiga #1187](https://tree.taiga.io/project/penpot/issue/1187)
+- Fix issue with multiple selection and groups [Taiga #1128](https://tree.taiga.io/project/penpot/issue/1128)
+- Fix issue with red handler indicator on resize [Taiga #1188](https://tree.taiga.io/project/penpot/issue/1188)
+- Fix show correct error when google auth is disabled [Taiga #1119](https://tree.taiga.io/project/penpot/issue/1119)
+- Fix text alignment in preview [#594](https://github.com/penpot/penpot/issues/594)
+- Fix unexpected exception when uploading image [Taiga #1120](https://tree.taiga.io/project/penpot/issue/1120)
+- Fix updates on collaborative editing not updating selection rectangles [Taiga #1127](https://tree.taiga.io/project/penpot/issue/1127)
+- Make the team deletion deferred (in the same way other objects)
+
+### :heart: Community contributions by (Thank you!)
+
+- abtinmo [#538](https://github.com/penpot/penpot/pull/538)
+- kdrag0n [#585](https://github.com/penpot/penpot/pull/585)
+- nisrulz [#586](https://github.com/penpot/penpot/pull/586)
+- tomer [#575](https://github.com/penpot/penpot/pull/575)
+- violoncelloCH [#554](https://github.com/penpot/penpot/pull/554)
+
+## 1.1.0-alpha
+
+- Bugfixing and stabilization post-launch
+- Some changes to the register flow
+- Improved MacOS shortcuts and helpers
+- Small changes to shape creation
+
+## 1.0.0-alpha
+
+Initial release

CONTRIBUTING.md

@@ -1,8 +1,8 @@
 # Contributing Guide #
 
 Thank you for your interest in contributing to Penpot. This is a
-generic guide that details how to contribute to Penpot in a way that is
-efficient for everyone. If you want a specific documentation for
+generic guide that details how to contribute to Penpot in a way that
+is efficient for everyone. If you want a specific documentation for
 different parts of the platform, please refer to `docs/` directory.
 
 
@@ -19,12 +19,20 @@ If you found a bug, please report it, as far as possible with:
 - a browser and the browser version used
 - a dev tools console exception stack trace (if it is available)
 
+If you found a bug that you consider better discuss in private (for
+example: security bugs), consider first send an email to
+`support@penpot.app`.
+
+**We don't have formal bug bounty program for security reports; this
+is an open source application and your contribution will be recognized
+in the changelog.**
+
 
 ## Pull requests ##
 
 If you want propose a change or bug fix with the Pull-Request system
-firstly you should carefully read the **Contributor License Aggreement**
-section and format your commits accordingly.
+firstly you should carefully read the **DCO** section and format your
+commits accordingly.
 
 If you intend to fix a bug it's fine to submit a pull request right
 away but we still recommend to file an issue detailing what you're
@@ -46,7 +54,7 @@ We will use the `easy fix` mark for tag for indicate issues that are
 easy for beginners.
 
 
-## Commit Message Guidelines ##
+## Commit Guidelines ##
 
 We have very precise rules over how our git commit messages can be formatted.
 
@@ -70,7 +78,6 @@ Where type is:
 - :ambulance: `:ambulance:` a commit that fixes critical bug
 - :books: `:books:` a commit that improves or adds documentation
 - :construction: `:construction:`: a wip commit
-- :construction_worker: `:construction_worker:` a commit with CI related stuff
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
@@ -83,13 +90,14 @@ More info:
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
-The subject should be:
-
-- Use the imperative mood.
-- Capitalize the first letter.
-- Don't put a period at the end of the subject line.
-- Put a blank line between the subject line and the body.
+Each commit should have:
 
+- A concise subject using imperative mood.
+- The subject should have capitalized the first letter and without
+  period at the end.
+- A blank line between the subject line and the body.
+- An entry on the CHANGES.md file if applicable, referencing the
+  github or taiga issue/user-story using the these same rules.
 
 ## Code of conduct ##
 
@@ -127,7 +135,7 @@ This Code of Conduct is adapted from the Contributor Covenant, version
 1.1.0, available from http://contributor-covenant.org/version/1/1/0/
 
 
-## Contributor License Agreement ##
+## Developer's Certificate of Origin (DCO) ##
 
 By submitting code you are agree and can certify the below:
 
@@ -157,9 +165,9 @@ By submitting code you are agree and can certify the below:
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
 
-Then, all your patches should contain a sign-off at the end of the
-patch/commit description body. It can be automatically added on adding
-`-s` parameter to `git commit`.
+Then, all your code patches (**documentation are excluded**) should
+contain a sign-off at the end of the patch/commit description body. It
+can be automatically added on adding `-s` parameter to `git commit`.
 
 This is an example of the aspect of the line:
 

README.md

@@ -2,36 +2,60 @@
 [uri_license]: https://www.mozilla.org/en-US/MPL/2.0
 [uri_license_image]: https://img.shields.io/badge/MPL-2.0-blue.svg
 
-[![License: MPL-2.0][uri_license_image]][uri_license]
-[![Gitter](https://badges.gitter.im/sereno-xyz/community.svg)](https://gitter.im/penpot/community)
-[![Managed with Taiga.io](https://img.shields.io/badge/managed%20with-TAIGA.io-709f14.svg)](https://tree.taiga.io/project/uxboxproject/ "Managed with Taiga.io")
+<h1 align="center">
+  <br>
+  <img src="https://penpot.app/images/readme/readme-logo.jpg" alt="PENPOT">
+</h1>
+
+<p align="center"><a href="https://www.mozilla.org/en-US/MPL/2.0" rel="nofollow"><img src="https://camo.githubusercontent.com/3fcf3d6b678ea15fde3cf7d6af0e242160366282d62a7c182d83a50bfee3f45e/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f4d504c2d322e302d626c75652e737667" alt="License: MPL-2.0" data-canonical-src="https://img.shields.io/badge/MPL-2.0-blue.svg" style="max-width:100%;"></a>
+<a href="https://gitter.im/penpot/community" rel="nofollow"><img src="https://camo.githubusercontent.com/5b0aecb33434f82a7b158eab7247544235ada0cf7eeb9ce8e52562dd67f614b7/68747470733a2f2f6261646765732e6769747465722e696d2f736572656e6f2d78797a2f636f6d6d756e6974792e737667" alt="Gitter" data-canonical-src="https://badges.gitter.im/sereno-xyz/community.svg" style="max-width:100%;"></a>
+<a href="https://tree.taiga.io/project/penpot/" title="Managed with Taiga.io" rel="nofollow"><img src="https://camo.githubusercontent.com/4a1d1112f0272e3393b1e8da312ff4435418e9e2eb4c0964881e3680f90a653c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6d616e61676564253230776974682d54414947412e696f2d3730396631342e737667" alt="Managed with Taiga.io" data-canonical-src="https://img.shields.io/badge/managed%20with-TAIGA.io-709f14.svg" style="max-width:100%;"></a>
+<a href="https://gitpod.io/#https://github.com/penpot/penpot" rel="nofollow"><img src="https://camo.githubusercontent.com/daadb4894128d1e19b72d80236f5959f1f2b47f9fe081373f3246131f0189f6c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f476974706f642d72656164792d2d746f2d2d636f64652d626c75653f6c6f676f3d676974706f64" alt="Gitpod ready-to-code" data-canonical-src="https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod" style="max-width:100%;"></a></p>
+
+![PENPOT](https://penpot.app/images/readme/home-ui.jpg)
 
 
-# PENPOT #
+## What is Penpot? ##
 
-We’re excited to share that Uxbox is now Penpot! We’re changing the name, but keeping the same project essence. Stay in the loop for more news coming early 2021. Alpha release is close!
+Penpot is the first **Open Source design** and prototyping platform meant for cross-domain teams. Non dependent on operating systems, Penpot is web based and works with open web standards (SVG). For all and empowered by the community.
 
-![PENPOT](https://raw.githubusercontent.com/penpot/penpot/develop/docs/screenshot.png)
+- [How to use](#how-to-use)
+- [Help center](#help-center)
+- [Contributing](#contributing)
+- [Give feedback](#give-feedback)
+- [Tutorials](#tutorials)
+- [License](#license)
 
+## How to use ##
 
-## Introduction ##
+Login or Register on our Penpot cloud app. Create a team to work together on projects and share design assets or jump right away into Penpot and **start designing** by your own.
 
-The open-source solution for design and prototyping. PENPOT is
-currently at an early development stage but we are working hard to
-bring you the beta version as soon as possible. Follow the project
-progress in Twitter or Github and stay tuned!
+✏️ [Start using Penpot](https://design.penpot.app)
 
+You can also install Penpot in a local environment. This section details everything you need to know to get Penpot up and running in production environments. Although it can be installed in many ways, the recommended approach is using **docker** and **docker-compose**.
 
-## SVG based ##
+🐳 [Install docker](https://help.penpot.app/technical-guide/getting-started/)
 
-Penpot works with SVG, a standard format, for all your designs and
-prototypes . This means that all your stuff in Penpot is portable and
-editable in many other vector tools and easy to use on the web.
+## Help center ##
 
-[See SVG specification](https://www.w3.org/Graphics/SVG/)
+In this documentation you will find (almost) everything you need to know about how to work with Penpot. From the interface basics to advanced functionality.
+
+📖 [User guide](https://help.penpot.app/user-guide/)
+
+❓ [FAQs](https://help.penpot.app/faqs/)
+
+🖥️ [Technical guide](https://help.penpot.app/technical-guide/)
+
+❤️ [Contributing guide](https://help.penpot.app/contributing-guide/)
+
+![User guide](https://penpot.app/images/readme/help-center.jpg)
 
 ## Contributing ##
 
+<p align="center">
+  <img src="https://penpot.app/images/open-source.png" alt="Open Source">
+</p>
+
 **Open to you!**
 
 We love the open source software community. Contributing is our
@@ -40,11 +64,24 @@ and improve Penpot. All your awesome ideas and code are welcome!
 
 Please refer to the [Contributing Guide](./CONTRIBUTING.md)
 
+## Give feedback ##
 
-## Documentation ##
+You can ask and answer questions, have open-ended conversations, and follow along on decisions affecting the project.
 
-Please refer to [docs/ directory](./docs/).
+✉️ [Mail us](mailto:info@penpot.app)
 
+💬 [GitHub discussions](https://github.com/penpot/penpot/discussions)
+
+🐞 [GitHub issues](https://github.com/penpot/penpot/issues)
+
+✍️️ [Gitter](https://gitter.im/penpot/community)
+
+## Tutorials ##
+
+You can ask and answer questions, have open-ended conversations, and follow along on decisions affecting the project.
+Would you like to know more about Penpot? We recommend you to visit our youtube channel and learn more about the functionalities and possibilities of Penpot with our video tutorials.
+
+🎞️ [YouTube channel](https://www.youtube.com/channel/UCAqS8G72uv9P5HG1IfgnQ9g)
 
 ## License ##
 
@@ -52,4 +89,6 @@ Please refer to [docs/ directory](./docs/).
 This Source Code Form is subject to the terms of the Mozilla Public
 License, v. 2.0. If a copy of the MPL was not distributed with this
 file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Copyright (c) UXBOX Labs SL
 ```

backend/build.clj

@@ -0,0 +1,36 @@
+(ns build
+  (:refer-clojure :exclude [compile])
+  (:require
+   [clojure.tools.build.api :as b]
+   [clojure.java.io]))
+
+(def class-dir "target/classes")
+(def basis (b/create-basis {:project "deps.edn"}))
+(def jar-file "target/penpot.jar")
+
+(defn clean [_]
+  (b/delete {:path "target"}))
+
+(defn jar [_]
+  (b/copy-dir
+   {:src-dirs ["src" "resources"]
+    :target-dir class-dir})
+
+  (b/compile-clj
+   {:basis basis
+    :src-dirs ["src"]
+    :class-dir class-dir})
+
+  (b/uber
+   {:class-dir class-dir
+    :uber-file jar-file
+    :main 'clojure.main
+    :exclude [#"goog.*" #"^javasist.*"]
+    :basis basis}))
+
+(defn compile [_]
+  (b/javac
+   {:src-dirs ["dev/java"]
+    :class-dir class-dir
+    :basis basis
+    :javac-opts ["-source" "11" "-target" "11"]}))

backend/deps.edn

@@ -1,98 +1,80 @@
-{:mvn/repos
- {"central" {:url "https://repo1.maven.org/maven2/"}
-  "clojars" {:url "https://clojars.org/repo"}
-  "jcenter" {:url "https://jcenter.bintray.com/"}}
- :deps
- {org.clojure/clojure {:mvn/version "1.10.2"}
-  org.clojure/clojurescript {:mvn/version "1.10.773"}
-  org.clojure/data.json {:mvn/version "1.0.0"}
-  org.clojure/core.async {:mvn/version "1.3.610"}
+{:deps
+ {penpot/common {:local/root "../common"}
+  org.clojure/core.async {:mvn/version "1.5.648"}
 
   ;; Logging
-  org.clojure/tools.logging {:mvn/version "1.1.0"}
-  org.apache.logging.log4j/log4j-api {:mvn/version "2.14.0"}
-  org.apache.logging.log4j/log4j-core {:mvn/version "2.14.0"}
-  org.apache.logging.log4j/log4j-web {:mvn/version "2.14.0"}
-  org.apache.logging.log4j/log4j-jul {:mvn/version "2.14.0"}
-  org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.14.0"}
-  org.slf4j/slf4j-api {:mvn/version "1.7.30"}
+  org.zeromq/jeromq {:mvn/version "0.5.2"}
 
-  org.graalvm.js/js {:mvn/version "20.3.0"}
+  com.taoensso/nippy {:mvn/version "3.1.1"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.1-1"}
+  org.clojure/data.fressian {:mvn/version "1.0.0"}
 
-  io.prometheus/simpleclient {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_hotspot {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_jetty {:mvn/version "0.9.0"
+  io.prometheus/simpleclient {:mvn/version "0.14.1"}
+  io.prometheus/simpleclient_hotspot {:mvn/version "0.14.1"}
+  io.prometheus/simpleclient_jetty {:mvn/version "0.14.1"
                                     :exclusions [org.eclipse.jetty/jetty-server
                                                  org.eclipse.jetty/jetty-servlet]}
-  io.prometheus/simpleclient_httpserver {:mvn/version "0.9.0"}
+  io.prometheus/simpleclient_httpserver {:mvn/version "0.14.1"}
 
-  selmer/selmer {:mvn/version "1.12.33"}
-  expound/expound {:mvn/version "0.8.7"}
-  com.cognitect/transit-clj {:mvn/version "1.0.324"}
+  io.lettuce/lettuce-core {:mvn/version "6.1.6.RELEASE"}
+  java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
-  io.lettuce/lettuce-core {:mvn/version "5.2.2.RELEASE"}
-  java-http-clj/java-http-clj {:mvn/version "0.4.1"}
+  funcool/yetti {:git/tag "v4.0" :git/sha "59ed2a7"
+                 :git/url "https://github.com/funcool/yetti.git"
+                 :exclusions [org.slf4j/slf4j-api]}
 
-  info.sunng/ring-jetty9-adapter {:mvn/version "0.14.2"}
-  seancorfield/next.jdbc {:mvn/version "1.1.613"}
-  metosin/reitit-ring {:mvn/version "0.5.11"}
-  metosin/jsonista {:mvn/version "0.3.1"}
+  com.github.seancorfield/next.jdbc {:mvn/version "1.2.761"}
+  metosin/reitit-ring {:mvn/version "0.5.15"}
+  org.postgresql/postgresql {:mvn/version "42.3.1"}
+  com.zaxxer/HikariCP {:mvn/version "5.0.1"}
+  funcool/datoteka {:mvn/version "2.0.0"}
 
-  org.postgresql/postgresql {:mvn/version "42.2.18"}
-  com.zaxxer/HikariCP {:mvn/version "3.4.5"}
+  buddy/buddy-hashers {:mvn/version "1.8.158"}
+  buddy/buddy-sign {:mvn/version "3.4.333"}
 
-  funcool/log4j2-clojure {:mvn/version "2020.11.23-1"}
-  funcool/datoteka {:mvn/version "1.2.0"}
-  funcool/promesa {:mvn/version "6.0.0"}
-  funcool/cuerdas {:mvn/version "2020.03.26-3"}
-
-  buddy/buddy-core {:mvn/version "1.9.0"}
-  buddy/buddy-hashers {:mvn/version "1.7.0"}
-  buddy/buddy-sign {:mvn/version "3.3.0"}
-
-  lambdaisland/uri {:mvn/version "1.4.54"
-                    :exclusions [org.clojure/data.json]}
-
-  frankiesardo/linked {:mvn/version "1.3.0"}
-  danlentz/clj-uuid {:mvn/version "0.1.9"}
-  org.jsoup/jsoup {:mvn/version "1.13.1"}
+  org.jsoup/jsoup {:mvn/version "1.14.3"}
   org.im4java/im4java {:mvn/version "1.4.0"}
-  org.lz4/lz4-java {:mvn/version "1.7.1"}
-  commons-io/commons-io {:mvn/version "2.8.0"}
-  org.apache.commons/commons-pool2 {:mvn/version "2.9.0"}
-  com.sun.mail/jakarta.mail {:mvn/version "2.0.0"}
+  org.lz4/lz4-java {:mvn/version "1.8.0"}
 
-  puppetlabs/clj-ldap {:mvn/version"0.3.0"}
+  org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
   integrant/integrant {:mvn/version "0.8.0"}
 
-  software.amazon.awssdk/s3 {:mvn/version "2.15.73"}
+  io.sentry/sentry {:mvn/version "5.5.2"}
 
-  ;; exception printing
-  io.aviso/pretty {:mvn/version "0.1.37"}
-  environ/environ {:mvn/version "1.2.0"}}
- :paths ["src" "resources" "../common" "common"]
+  ;; Pretty Print specs
+  pretty-spec/pretty-spec {:mvn/version "0.1.4"}
+  software.amazon.awssdk/s3 {:mvn/version "2.17.111"}}
+
+ :paths ["src" "resources" "target/classes"]
  :aliases
  {:dev
   {:extra-deps
-   {com.bhauman/rebel-readline {:mvn/version "0.1.4"}
-    org.clojure/tools.namespace {:mvn/version "1.1.0"}
-    org.clojure/test.check {:mvn/version "1.1.0"}
+   {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
+    org.clojure/tools.namespace {:mvn/version "RELEASE"}
+    org.clojure/test.check {:mvn/version "RELEASE"}
+    clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
+    org.clojure/data.csv {:mvn/version "RELEASE"}
+    com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
+    mockery/mockery {:mvn/version "RELEASE"}}
+   :extra-paths ["test" "dev"]}
 
-    fipp/fipp {:mvn/version "0.6.23"}
-    criterium/criterium {:mvn/version "0.4.6"}
-    mockery/mockery {:mvn/version "0.1.4"}}
-   :extra-paths ["tests" "dev"]}
+  :build
+  {:extra-deps {io.github.clojure/tools.build {:git/tag "v0.7.4" :git/sha "ac442da"}}
+   :ns-default build}
 
-  :fn-fixtures
-  {:exec-fn app.cli.fixtures/run
-   :args {}}
-
-  :tests
-  {:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.732"}}
+  :kaocha
+  {:extra-deps {lambdaisland/kaocha {:mvn/version "RELEASE"}}
    :main-opts ["-m" "kaocha.runner"]}
 
+  :test
+  {:extra-paths ["test"]
+   :extra-deps
+   {io.github.cognitect-labs/test-runner
+    {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
+   :exec-fn cognitect.test-runner.api/test}
+
   :outdated
-  {:extra-deps {antq/antq {:mvn/version "RELEASE"}}
+  {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
    :main-opts ["-m" "antq.core"]}
 
   :jmx-remote

backend/dev/user.clj

@@ -2,57 +2,43 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) UXBOX Labs SL
 
 (ns user
   (:require
+   [app.common.exceptions :as ex]
+   [app.common.geom.matrix :as gmt]
+   [app.common.perf :as perf]
+   [app.common.transit :as t]
    [app.config :as cfg]
    [app.main :as main]
+   [app.util.blob :as blob]
+   [app.util.fressian :as fres]
+   [app.util.json :as json]
    [app.util.time :as dt]
-   [app.util.transit :as t]
-   [app.common.exceptions :as ex]
-   [clojure.data.json :as json]
+   [clj-async-profiler.core :as prof]
+   [clojure.contrib.humanize :as hum]
    [clojure.java.io :as io]
-   [clojure.test :as test]
-   [clojure.pprint :refer [pprint]]
+   [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]
    [clojure.spec.alpha :as s]
    [clojure.spec.gen.alpha :as sgen]
    [clojure.test :as test]
+   [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
-   [criterium.core :refer [quick-bench bench with-progress-reporting]]
+   [datoteka.core]
    [integrant.core :as ig]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
+(set! *warn-on-reflection* true)
 
 (defonce system nil)
 
-;; --- Benchmarking Tools
-
-(defmacro run-quick-bench
-  [& exprs]
-  `(with-progress-reporting (quick-bench (do ~@exprs) :verbose)))
-
-(defmacro run-quick-bench'
-  [& exprs]
-  `(quick-bench (do ~@exprs)))
-
-(defmacro run-bench
-  [& exprs]
-  `(with-progress-reporting (bench (do ~@exprs) :verbose)))
-
-(defmacro run-bench'
-  [& exprs]
-  `(bench (do ~@exprs)))
-
 ;; --- Development Stuff
 
 (defn- run-tests
-  ([] (run-tests #"^app.tests.*"))
+  ([] (run-tests #"^app.*-test$"))
   ([o]
    (repl/refresh)
    (cond
@@ -69,7 +55,7 @@
   []
   (alter-var-root #'system (fn [sys]
                              (when sys (ig/halt! sys))
-                             (-> (main/build-system-config cfg/config)
+                             (-> main/system-config
                                  (ig/prep)
                                  (ig/init))))
   :started)
@@ -90,3 +76,19 @@
   []
   (stop)
   (repl/refresh-all :after 'user/start))
+
+(defn compression-bench
+  [data]
+  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))]
+    (print-table
+     [{:v1 (humanize (alength (blob/encode data {:version 1})))
+       :v2 (humanize (alength (blob/encode data {:version 2})))
+       :v3 (humanize (alength (blob/encode data {:version 3})))
+       :v4 (humanize (alength (blob/encode data {:version 4})))
+       }])))
+
+(defonce debug-tap
+  (do
+    (add-tap #(locking debug-tap
+                (prn "tap debug:" %)))
+    1))

backend/resources/api-doc.css

@@ -0,0 +1,101 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+pre {
+  margin: 0px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+  padding-top: 20px;
+  padding-bottom: 20px;
+  display: flex;
+  justify-content: center;
+}
+
+main {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  min-width: 900px;
+  width: 900px;
+}
+
+header {
+  border-bottom: 1px solid #c0c0c0;
+  display: flex;
+  justify-content: center;
+  width: 100%;
+}
+
+.rpc-doc-content {
+  margin-top: 20px;
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  /* border: 1px solid red; */
+  padding: 5px;
+}
+
+.rpc-doc-content > h2:not(:first-child) {
+  margin-top: 30px;
+}
+
+
+.rpc-items {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+}
+
+.rpc-item {
+  /* border: 1px solid red; */
+  cursor: pointer;
+  display: flex;
+  flex-direction: column;
+}
+
+.rpc-item:not(:last-child) {
+  margin-bottom: 3px;
+}
+
+.rpc-row-info {
+  cursor: pointer;
+  display: flex;
+  background-color: #eeeeee;
+  padding: 5px 10px;
+}
+
+.rpc-row-info > *:not(:last-child) {
+  margin-right: 10px;
+}
+
+.rpc-row-info > * {
+  /* border: 1px solid green; */
+}
+
+.rpc-row-info > .type {
+  font-weight: bold;
+  width: 70px;
+}
+
+.rpc-row-info > .name {
+  width: 280px;
+  /* font-weight: bold; */
+}
+
+.rpc-row-info > .tags > .tag > span:first-child {
+  font-weight: bold;
+}
+
+.hidden {
+  display: none;
+}
+
+.rpc-row-detail {
+  padding: 5px 10px;
+  padding-bottom: 20px;
+}

backend/resources/api-doc.js

@@ -0,0 +1,27 @@
+(function() {
+  document.addEventListener("DOMContentLoaded", function(event) {
+    const rows = document.querySelectorAll(".rpc-row-info");
+
+    const onRowClick = (event) => {
+      const target = event.currentTarget;
+      for (let node of rows) {
+        if (node !== target) {
+          node.nextElementSibling.classList.add("hidden");
+        } else {
+          const sibling = target.nextElementSibling;
+
+          if (sibling.classList.contains("hidden")) {
+            sibling.classList.remove("hidden");
+          } else {
+            sibling.classList.add("hidden");
+          }
+        }
+      }
+    };
+
+    for (let node of rows) {
+      node.addEventListener("click", onRowClick);
+    }
+
+  });
+})();

backend/resources/api-doc.tmpl

@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Builtin API Documentation - Penpot</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      {% include "api-doc.css" %}
+    </style>
+    <script>
+      {% include "api-doc.js" %}
+    </script>
+  </head>
+  <body>
+    <main>
+      <header>
+        <h1>Penpot API Documentation</h1>
+      </header>
+      <section class="rpc-doc-content">
+
+        <h2>RPC QUERY METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in query-methods %}
+            <li class="rpc-item">
+              <div class="rpc-row-info">
+                {# <div class="type">{{item.type}}</div> #}
+                <div class="name">{{item.name}}</div>
+                <div class="tags">
+                  <span class="tag">
+                    <span>Auth:</span>
+                    <span>{% if item.auth %}YES{% else %}NO{% endif %}</span>
+                  </span>
+                </div>
+              </div>
+              <div class="rpc-row-detail hidden">
+                {% if item.docs %}
+                  <h3>DOCSTRING:</h3>
+                  <p>{{item.docs}}</p>
+                {% endif %}
+
+                <h3>SPEC EXPLAIN:</h3>
+                <pre>{{item.spec}}</pre>
+              </div>
+            </li>
+          {% endfor %}
+        </ul>
+
+        <h2>RPC MUTATION METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in mutation-methods %}
+            <li class="rpc-item">
+              <div class="rpc-row-info">
+                {# <div class="type">{{item.type}}</div> #}
+                <div class="name">{{item.name}}</div>
+                <div class="tags">
+                  <span class="tag">
+                    <span>Auth:</span>
+                    <span>{% if item.auth %}YES{% else %}NO{% endif %}</span>
+                  </span>
+                </div>
+              </div>
+              <div class="rpc-row-detail hidden">
+                {% if item.docs %}
+                  <h3>DOCSTRING:</h3>
+                  <p>{{item.docs}}</p>
+                {% endif %}
+
+                <h3>SPEC EXPLAIN:</h3>
+                <pre>{{item.spec}}</pre>
+              </div>
+            </li>
+          {% endfor %}
+        </ul>
+      </section>
+    </main>
+  </body>
+</html>
+

backend/resources/builtin.edn

@@ -1,11 +0,0 @@
-{:icons
- [{:name "Material Design (Action)"
-   :path "./material/action/svg/production"
-   :regex #"^.*_48px\.svg$"}]
-
- :images
- [{:name "Generic Collection 1"
-   :path "./my-images/collection1/"
-   :regex #"^.*\.(png|jpg|webp)$"}]}
-
-

backend/resources/config/default.edn

@@ -1,44 +0,0 @@
-{;; A secret key used for create tokens
- ;; WARNING: this is a default secret key and
- ;; it should be overwritten in production env.
- :secret "5qjiAn-QUpawUNqGP10UZKklSqbLKcdGY3sJpq0UUACpVXGg2HOFJCBejDWVHskhRyp7iHb4rjOLXX2ZjF-5cw"
- 
- :registration
- {
-  :enabled true}
- 
- :smtp
- {:host "localhost" ;; Hostname of the desired SMTP server.
-  :port 25          ;; Port of SMTP server.
-  :user nil         ;; Username to authenticate with (if authenticating).
-  :pass nil         ;; Password to authenticate with (if authenticating).
-  :ssl false        ;; Enables SSL encryption if value is truthy.
-  :tls false        ;; Enables TLS encryption if value is truthy.
-  :enabled false    ;; Enables SMTP if value is truthy.
-  :noop true}
-
- :auth-options {:alg :a256kw :enc :a128cbc-hs256}
-
- :email {:reply-to "no-reply@uxbox.io"
-         :from "no-reply@uxbox.io"
-         :support "support@uxbox.io"}
-
- :http {:port 6060
-        :max-body-size 52428800
-        :debug true}
-
- :media
- {:directory "resources/public/media"
-  :uri "http://localhost:6060/media/"}
-
- :static
- {:directory "resources/public/static"
-  :uri "http://localhost:6060/static/"}
-
- :database
- {:adapter "postgresql"
-  :username nil
-  :password nil
-  :database-name "uxbox"
-  :server-name "localhost"
-  :port-number 5432}}

backend/resources/config/test.edn

@@ -1,18 +0,0 @@
-{:migrations
- {:verbose false}
-
- :media
- {:directory "/tmp/uxbox/media"
-  :uri "http://localhost:6060/media/"}
-
- :static
- {:directory "/tmp/uxbox/static"
-  :uri "http://localhost:6060/static/"}
-
- :database
- {:adapter "postgresql"
-  :username nil
-  :password nil
-  :database-name "test"
-  :server-name "localhost"
-  :port-number 5432}}

backend/resources/emails-mjml/change-email/en.mjml

@@ -30,14 +30,14 @@
           for security reasons.
         </mj-text>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -57,7 +57,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/invite-to-team/en.mjml

@@ -23,14 +23,14 @@
           Accept invite
         </mj-button>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -50,7 +50,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/password-recovery/en.mjml

@@ -32,14 +32,14 @@
           it. Your password won't be changed.
         </mj-text>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -59,7 +59,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/register/en.mjml

@@ -21,22 +21,22 @@
       <mj-column>
         <mj-text font-size="24px" font-weight="600">Hello {{name}}!</mj-text>
         <mj-text>
-          Thanks for signing up for your UXBOX account! Please verify your
-          email using the link below adn get started building mockups and
+          Thanks for signing up for your Penpot account! Please verify your
+          email using the link below and get started building mockups and
           prototypes today!
         </mj-text>
         <mj-button href="{{ public-uri }}/#/auth/verify-token?token={{token}}">
           Verify email
         </mj-button>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -56,7 +56,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails/change-email/en.txt

@@ -10,4 +10,4 @@ If you received this email by mistake, please consider changing your password
 for security reasons.
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/feedback/en.html

@@ -0,0 +1,45 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <title></title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  </head>
+  <body>
+    <p>
+      <strong>Feedback from:</strong><br />
+      {% if profile %}
+        <span>
+          <span>Name: </span>
+          <span><code>{{profile.fullname}}</code></span>
+        </span>
+        <br />
+
+        <span>
+          <span>Email: </span>
+          <span>{{profile.email}}</span>
+        </span>
+        <br />
+
+        <span>
+          <span>ID: </span>
+          <span><code>{{profile.id}}</code></span>
+        </span>
+      {% else %}
+        <span>
+          <span>Email: </span>
+          <span>{{profile.email}}</span>
+        </span>
+      {% endif %}
+    </p>
+    <p>
+      <strong>Subject:</strong><br />
+      <span>{{subject}}</span>
+    </p>
+
+    <p>
+      <strong>Message:</strong><br />
+      {{content|linebreaks-br|safe}}
+    </p>
+  </body>
+</html>

backend/resources/emails/feedback/en.subj

@@ -0,0 +1 @@
+[PENPOT FEEDBACK]: {{subject}}

backend/resources/emails/feedback/en.txt

@@ -0,0 +1,9 @@
+{% if profile %}
+Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+{% else %}
+Feedback from: {{email}}
+{% endif %}
+
+Subject: {{subject}}
+
+{{content}}

backend/resources/emails/invite-to-team/en.subj

@@ -1 +1 @@
-Inviation to join {{team}}
+Invitation to join {{team}}

backend/resources/emails/invite-to-team/en.txt

@@ -7,4 +7,4 @@ Accept invitation using this link:
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/password-recovery/en.txt

@@ -9,4 +9,4 @@ If you received this email by mistake, you can safely ignore it. Your password
 won't be changed.
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/register/en.html

@@ -173,7 +173,7 @@
                   </tr>
                   <tr>
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below adn get started building mockups and prototypes today!</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below and get started building mockups and prototypes today!</div>
                     </td>
                   </tr>
                   <tr>
@@ -465,4 +465,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/emails/register/en.txt

@@ -1,9 +1,9 @@
 Hello {{name}}!
 
-Thanks for signing up for your UXBOX account! Please verify your email using the
-link below adn get started building mockups and prototypes today!
+Thanks for signing up for your Penpot account! Please verify your email using the
+link below and get started building mockups and prototypes today!
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/error-report.tmpl

@@ -1,162 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>penpot - error report {{id}}</title>
-    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
-    <style>
-      body {
-        margin: 0px;
-        padding: 0px;
-      }
-      pre {
-        margin: 0px;
-      }
-      * {
-        font-family: "JetBrains Mono", monospace; 
-        font-size: 12px;
-      }
-      .table {
-        display: flex;
-        flex-direction: column;
-        margin: 10px;
-      }
-
-      .table-row {
-        display: flex;
-        /* width: 100%; */
-        /* border: 1px solid red; */
-      }
-
-      .table-key {
-        font-weight: 600;
-        width: 70px;
-        padding: 4px;
-      }
-
-      .table-val {
-        font-weight: 200;
-        color: #333;
-        padding: 4px;
-      }
-
-      .multiline {
-        margin-top: 15px;
-        flex-direction: column;
-      }
-
-      .multiline .table-key {
-        margin-bottom: 10px;
-        border-bottom: 1px dashed #dddddd;
-        /* padding: 4px; */
-        width: unset;
-      }
-
-    </style>
-  </head>
-  <body>
-    <div class="table">
-      <div class="table-row">
-        <div class="table-key" title="Error ID">ERID: </div>
-        <div class="table-val">{{id}}</div>
-      </div>
-      {% if profile-id %}
-      <div class="table-row">
-        <div class="table-key" title="Profile ID">PFID: </div>
-        <div class="table-val">{{profile-id}}</div>
-      </div>
-      {% endif %}
-
-      {% if user-agent %}
-      <div class="table-row">
-        <div class="table-key">UAGENT: </div>
-        <div class="table-val">{{user-agent}}</div>
-      </div>
-      {% endif %}
-
-      {% if frontend-version %}
-      <div class="table-row">
-        <div class="table-key">FVERS: </div>
-        <div class="table-val">{{frontend-version}}</div>
-      </div>
-      {% endif %}
-
-      <div class="table-row">
-        <div class="table-key">BVERS: </div>
-        <div class="table-val">{{version}}</div>
-      </div>
-
-      <div class="table-row">
-        <div class="table-key">HOST: </div>
-        <div class="table-val">{{host}}</div>
-      </div>
-
-      {% if type %}
-      <div class="table-row">
-        <div class="table-key">TYPE: </div>
-        <div class="table-val">{{type}}</div>
-      </div>
-      {% endif %}
-
-      {% if code %}
-      <div class="table-row">
-        <div class="table-key">CODE: </div>
-        <div class="table-val">{{code}}</div>
-      </div>
-      {% endif %}
-
-      <div class="table-row">
-        <div class="table-key">CLASS: </div>
-        <div class="table-val">{{class}}</div>
-      </div>
-
-      <div class="table-row">
-        <div class="table-key">HINT: </div>
-        <div class="table-val">{{hint}}</div>
-      </div>
-
-      {% if method %}
-      <div class="table-row">
-        <div class="table-key">PATH: </div>
-        <div class="table-val">{{method|upper}} {{path}}</div>
-      </div>
-      {% endif %}
-
-      {% if params %}
-      <div class="table-row multiline">
-        <div class="table-key">PARAMS: </div>
-        <div class="table-val">
-          <pre>{{params}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if explain %}
-      <div class="table-row multiline">
-        <div class="table-key">EXPLAIN: </div>
-        <div class="table-val">
-          <pre>{{explain}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if data %}
-      <div class="table-row multiline">
-        <div class="table-key">EDATA: </div>
-        <div class="table-val">
-          <pre>{{data}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      <div class="table-row multiline">
-        <div class="table-key">TRACE:</div>
-        <div class="table-val">
-          <pre>{{message}}</pre>
-        </div>
-      </div>
-    </div>
-  </body>
-</html>
-

backend/resources/log4j2-bundle.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="info" monitorInterval="60">
-  <Appenders>
-    <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%t] %level{length=1} %logger{36} - %msg%n"/>
-    </Console>
-  </Appenders>
-
-  <Loggers>
-    <Logger name="com.zaxxer.hikari" level="error" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="org.eclipse.jetty" level="info" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="app" level="debug" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Root level="info">
-      <AppenderRef ref="console"/>
-    </Root>
-  </Loggers>
-</Configuration>

backend/resources/log4j2-devenv.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="info" monitorInterval="30">
+  <Appenders>
+    <Console name="console" target="SYSTEM_OUT">
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
+    </Console>
+
+    <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
+      <Policies>
+        <SizeBasedTriggeringPolicy size="50M"/>
+      </Policies>
+      <DefaultRolloverStrategy max="9"/>
+    </RollingFile>
+
+    <JeroMQ name="zmq">
+      <Property name="endpoint">tcp://localhost:45556</Property>
+      <JsonLayout complete="false" compact="true" includeTimeMillis="true" stacktraceAsString="true" properties="true" />
+    </JeroMQ>
+  </Appenders>
+
+  <Loggers>
+    <Logger name="com.zaxxer.hikari" level="error"/>
+    <Logger name="io.lettuce" level="error" />
+    <Logger name="org.eclipse.jetty" level="error" />
+    <Logger name="org.postgresql" level="error" />
+
+    <Logger name="app.cli" level="debug" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+
+    <Logger name="app.loggers" level="debug" additivity="false">
+      <AppenderRef ref="main" level="debug" />
+    </Logger>
+
+    <Logger name="app" level="all" additivity="false">
+      <AppenderRef ref="main" level="trace" />
+      <AppenderRef ref="zmq" level="debug" />
+    </Logger>
+
+    <Logger name="penpot" level="debug" additivity="false">
+      <AppenderRef ref="main" level="debug" />
+      <AppenderRef ref="zmq" level="debug" />
+    </Logger>
+
+    <Logger name="user" level="trace" additivity="false">
+      <AppenderRef ref="main" level="trace" />
+    </Logger>
+
+    <Root level="info">
+      <AppenderRef ref="main" />
+    </Root>
+  </Loggers>
+</Configuration>

backend/resources/log4j2.xml

@@ -1,43 +1,25 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="info" monitorInterval="30">
+<Configuration status="info" monitorInterval="60">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
-
-    <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
-      <Policies>
-        <SizeBasedTriggeringPolicy size="50M"/>
-      </Policies>
-      <DefaultRolloverStrategy max="9"/>
-    </RollingFile>
-
-    <CljFn name="error-reporter" ns="app.error-reporter" fn="queue-fn">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
-    </CljFn>
   </Appenders>
 
   <Loggers>
-    <Logger name="com.zaxxer.hikari" level="error" additivity="false" />
-    <Logger name="org.eclipse.jetty" level="error" additivity="false" />
-    <Logger name="io.lettuce" level="error" additivity="false" />
-
-    <Logger name="app.cli" level="debug" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="app.error-reporter" level="debug" additivity="false">
-      <AppenderRef ref="main" level="debug" />
-    </Logger>
+    <Logger name="com.zaxxer.hikari" level="error" />
+    <Logger name="org.eclipse.jetty" level="error" />
 
     <Logger name="app" level="debug" additivity="false">
-      <AppenderRef ref="main" level="debug" />
-      <AppenderRef ref="error-reporter" level="error" />
+      <AppenderRef ref="console" />
+    </Logger>
+
+    <Logger name="penpot" level="fatal" additivity="false">
+      <AppenderRef ref="console" />
     </Logger>
 
     <Root level="info">
-      <AppenderRef ref="main" />
+      <AppenderRef ref="console" />
     </Root>
   </Loggers>
 </Configuration>

backend/resources/templates/base.tmpl

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>{% block title %}{% endblock %}</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      {% include "templates/styles.css"  %}
+    </style>
+  </head>
+  <body>
+    {% block content %}
+    {% endblock %}
+  </body>
+</html>
+

backend/resources/templates/debug.tmpl

@@ -0,0 +1,32 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+Debug Main Page
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Debug INDEX:</h1>
+  <div>[<a href="/dbg/error">ERRORS</a>]</div>
+</nav>
+<main class="index">
+  <section>
+    <h2>Download file data:</h2>
+    <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
+    <form method="get" action="/dbg/file/data">
+      <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+      <input type="hidden" name="download" value="1" />
+      <input type="submit" value="Download" />
+    </form>
+  </section>
+
+  <section>
+    <h2>Upload File Data:</h2>
+    <desc>Create a new file on your draft projects using the file downloaded from the previous section.</desc>
+    <form method="post" enctype="multipart/form-data" action="/dbg/file/data">
+      <input type="file" name="file" value="" />
+      <input type="submit" value="Upload" />
+    </form>
+  </section>
+</main>
+{% endblock %}

backend/resources/templates/error-list.tmpl

@@ -0,0 +1,18 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error list
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Latest error reports:</h1>
+</nav>
+<main class="horizontal-list">
+  <ul>
+    {% for item in items %}
+    <li><a href="/dbg/error/{{item.id}}">{{item.created-at}}</a></li>
+    {% endfor %}
+  </ul>
+</main>
+{% endblock %}

backend/resources/templates/error-report.tmpl

@@ -0,0 +1,98 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error report {{id}}
+{% endblock %}
+
+{% block content %}
+<nav>
+  <div>[<a href="/dbg/error">⮜</a>]</div>
+  <div>[<a href="#context">context</a>]</div>
+  <div>[<a href="#params">request params</a>]</div>
+  {% if data %}
+  <div>[<a href="#edata">error data</a>]</div>
+  {% endif %}
+  {% if spec-explain %}
+  <div>[<a href="#spec-explain">spec explain</a>]</div>
+  {% endif %}
+  {% if spec-problems %}
+  <div>[<a href="#spec-problems">spec problems</a>]</div>
+  {% endif %}
+  {% if spec-value %}
+  <div>[<a href="#spec-value">spec value</a>]</div>
+  {% endif %}
+
+  {% if trace %}
+  <div>[<a href="#trace">error trace</a>]</div>
+  {% endif %}
+</nav>
+<main>
+  <div class="table">
+    <div class="table-row multiline">
+      <div id="context" class="table-key">CONTEXT: </div>
+
+      <div class="table-val">
+        <h1>{{hint}}</h1>
+      </div>
+
+      <div class="table-val">
+        <pre>{{context}}</pre>
+      </div>
+    </div>
+
+    {% if params %}
+    <div class="table-row multiline">
+      <div id="params" class="table-key">REQUEST PARAMS: </div>
+      <div class="table-val">
+        <pre>{{params}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if data %}
+    <div class="table-row multiline">
+      <div id="edata" class="table-key">ERROR DATA: </div>
+      <div class="table-val">
+        <pre>{{data}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-explain %}
+    <div class="table-row multiline">
+      <div id="spec-explain" class="table-key">SPEC EXPLAIN: </div>
+      <div class="table-val">
+        <pre>{{spec-explain}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-problems %}
+    <div class="table-row multiline">
+      <div id="spec-problems" class="table-key">SPEC PROBLEMS: </div>
+      <div class="table-val">
+        <pre>{{spec-problems}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-value %}
+    <div class="table-row multiline">
+      <div id="spec-value" class="table-key">SPEC VALUE: </div>
+      <div class="table-val">
+        <pre>{{spec-value}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if trace %}
+    <div class="table-row multiline">
+      <div id="trace" class="table-key">TRACE:</div>
+      <div class="table-val">
+        <pre>{{trace}}</pre>
+      </div>
+    </div>
+    {% endif %}
+  </div>
+</main>
+{% endblock %}

backend/resources/templates/styles.css

@@ -0,0 +1,150 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+}
+
+pre {
+  margin: 0px;
+  line-height: 16px;
+}
+
+desc {
+  display: flex;
+  margin-bottom: 10px;
+  font-size: 10px;
+  color: #666;
+}
+
+input[type=text], input[type=submit] {
+  padding: 3px;
+}
+
+main {
+  margin: 20px;
+}
+
+nav {
+  position: fixed;
+  width: 100vw;
+  top: 0;
+  left: 0;
+  padding: 5px 20px;
+  display: flex;
+  background: #e3e3e3;
+}
+
+nav > h1 {
+  padding: 0px;
+  margin: 0px;
+  font-size: 11px;
+}
+
+nav > div {
+  text-transform: uppercase;
+  font-weight: bold;
+}
+
+nav > div:not(:last-child) {
+  margin-right: 10px;
+}
+
+.table {
+  margin-top: 25px;
+  display: flex;
+  flex-direction: column;
+}
+
+.table-row {
+  display: flex;
+  padding-bottom: 15px;
+  /* width: 100%; */
+  /* border: 1px solid red; */
+}
+
+.table-key {
+  font-weight: 600;
+  width: 60px;
+  padding: 4px;
+
+  padding-top: 40px;
+  margin-top: -40px;
+}
+
+.table-val {
+  font-weight: 200;
+  color: #333;
+  padding: 4px;
+}
+
+.multiline {
+  margin-top: 15px;
+  flex-direction: column;
+}
+
+.multiline .table-key {
+  margin-bottom: 10px;
+  border-bottom: 1px dashed #dddddd;
+  /* padding: 4px; */
+  width: unset;
+}
+
+.index {
+  margin-top: 40px;
+}
+
+.index > section {
+  padding: 10px;
+  background-color: #e3e3e3;
+}
+
+.index > section:not(:last-child) {
+  margin-bottom: 10px;
+}
+
+
+.index > section > h2 {
+  margin-top: 0px;
+}
+
+.horizontal-list {
+  margin: 20px;
+  margin-top: 40px;
+}
+
+.horizontal-list ul {
+  display: flex;
+  margin: 0px;
+  padding: 0px;
+  flex-direction: column;
+  flex-wrap: wrap;
+  height: calc(100vh - 75px);
+  justify-content: flex-start;
+}
+
+.horizontal-list li {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+  line-height: 18px;
+  min-width: 210px;
+  margin: 0px 20px;
+  cursor: pointer;
+  display: flex;
+  justify-content: center;
+  border-radius: 3px;
+}
+
+.horizontal-list li:hover {
+  background-color: #e9e9e9;
+}
+
+.horizontal-list li > a {
+  text-decoration: none;
+  color: inherit;
+}
+

backend/scripts/build

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+CURRENT_VERSION=$1;
+
+set -ex
+
+rm -rf target;
+mkdir -p target/classes;
+mkdir -p target/dist;
+echo "$CURRENT_VERSION" > target/classes/version.txt;
+
+clojure -T:build jar;
+mv target/penpot.jar target/dist/penpot.jar
+cp scripts/run.template.sh target/dist/run.sh;
+cp scripts/manage.template.sh target/dist/manage.sh;
+chmod +x target/dist/run.sh;
+chmod +x target/dist/manage.sh;
+
+
+

backend/scripts/build.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-CLASSPATH=`(clojure -Spath)`
-NEWCP="./main:./common"
-
-rm -rf ./target/dist
-mkdir -p ./target/dist/deps
-
-for item in $(echo $CLASSPATH | tr ":" "\n"); do
-    if [ "${item: -4}" == ".jar" ]; then
-        cp $item ./target/dist/deps/;
-        BN="$(basename -- $item)"
-        NEWCP+=":./deps/$BN"
-    fi
-done
-
-cp ./resources/log4j2-bundle.xml ./target/dist/log4j2.xml
-cp -r ./src ./target/dist/main
-cp -r ./resources/emails ./target/dist/main/
-cp -r ./resources/svgclean.js ./target/dist/main/
-cp -r ./resources/error-report.tmpl ./target/dist/main/
-cp -r ../common ./target/dist/common
-
-echo $NEWCP > ./target/dist/classpath;
-
-tee -a ./target/dist/run.sh  >> /dev/null <<EOF
-#!/usr/bin/env bash
-
-CP="$NEWCP"
-
-# Exports
-
-# Find java executable
-set +e
-JAVA_CMD=\$(type -p java)
-
-set -e
-if [[ ! -n "\$JAVA_CMD" ]]; then
-  if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="\$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
-fi
-
-if [ -f ./environ ]; then
-   source ./environ
-fi
-
-set -x
-exec \$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main
-EOF
-
-chmod +x ./target/dist/run.sh

backend/scripts/import-generic-collections.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-clojure -Adev -m app.cli.collimp $@
-

backend/scripts/manage.template.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m app.cli.manage "$@"

backend/scripts/prepare-release.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-if [ "$#" -e 0 ]; then
-    echo "Expecting parameters: 1=path to backend; 2=destination directory"
-    exit 1
-fi
-
-rm -rf $2 || exit 1;
-
-rsync -avr \
-      --exclude="/test" \
-      --exclude="/resources/public/media" \
-      --exclude="/target" \
-      --exclude="/scripts" \
-      --exclude="/.*" \
-      $1 $2;

backend/scripts/psql.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-PGPASSWORD=$PENPOT_DATABASE_PASSWORD psql $PENPOT_DATABASE_URI -U $PENPOT_DATABASE_USERNAME

backend/scripts/repl

@@ -1,7 +1,28 @@
 #!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
+# export PENPOT_DATABASE_USERNAME="penpot"
+# export PENPOT_DATABASE_PASSWORD="penpot"
+# export PENPOT_DATABASE_READONLY=true
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
+# export PENPOT_DATABASE_USERNAME="penpot_pre"
+# export PENPOT_DATABASE_PASSWORD="penpot_pre"
+# export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS"
+
+export OPTIONS="
+       -A:dev \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -J-XX:+UseZGC \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-Xms50m -J-Xmx1024m \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints";
+
+export OPTIONS_EVAL="nil"
+# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
 
 set -ex
-# clojure -Ojmx-remote -A:dev -e "(set! *warn-on-reflection* true)" -m rebel-readline.main
-clojure -A:jmx-remote:dev -J-Xms512m -J-Xmx512m -M -m rebel-readline.main
+exec clojure $OPTIONS -M -e "$OPTIONS_EVAL" -m rebel-readline.main

backend/scripts/run-tests-in-docker.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-clojure -Adev -m app.tests.main;

backend/scripts/run.template.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+set -x
+exec $JAVA_CMD $JVM_OPTS "$@" -jar penpot.jar -m app.main

backend/scripts/smtpd.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-python -m smtpd -n -c DebuggingServer localhost:25

backend/scripts/start-dev

@@ -1,15 +1,23 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+export PENPOT_FLAGS="$PENPOT_FLAGS enable-asserts"
 
 set -ex
 
-if [ ! -e ~/.fixtures-loaded ]; then
-    echo "Loading fixtures..."
-    clojure -Adev -X:fn-fixtures
-    touch ~/.fixtures-loaded
+if [ "$1" = "--watch" ]; then
+    echo "Start Watch..."
+
+    clojure -A:dev -M -m app.main &
+    PID=$!
+
+    npx nodemon \
+        --watch src \
+        --watch ../common \
+        --ext "clj" \
+        --signal SIGKILL \
+        --exec 'echo "(user/restart)" | nc -N localhost 6062'
+
+    kill -9 $PID
+else
+    clojure -A:dev -M -m app.main
 fi
-
-clojure -A:dev -M -m app.main
-
-

backend/scripts/tests.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env sh
-exec clojure -M:dev:tests "$@"

backend/src/app/cli/fixtures.clj

@@ -1,242 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.cli.fixtures
-  "A initial fixtures."
-  (:require
-   [app.common.pages :as cp]
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.main :as main]
-   [app.rpc.mutations.profile :as profile]
-   [app.util.blob :as blob]
-   [buddy.hashers :as hashers]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]))
-
-(defn- mk-uuid
-  [prefix & args]
-  (uuid/namespaced uuid/zero (apply str prefix (interpose "-" args))))
-
-;; --- Profiles creation
-
-(def password (hashers/derive "123123"))
-
-(def preset-small
-  {:num-teams 5
-   :num-profiles 5
-   :num-profiles-per-team 5
-   :num-projects-per-team 5
-   :num-files-per-project 5
-   :num-draft-files-per-profile 10})
-
-(defn- rng-ids
-  [rng n max]
-  (let [stream (->> (.longs rng 0 max)
-                    (.iterator)
-                    (iterator-seq))]
-    (reduce (fn [acc item]
-              (if (= (count acc) n)
-                (reduced acc)
-                (conj acc item)))
-            #{}
-            stream)))
-
-(defn- rng-vec
-  [rng vdata n]
-  (let [ids (rng-ids rng n (count vdata))]
-    (mapv #(nth vdata %) ids)))
-
-(defn- rng-nth
-  [rng vdata]
-  (let [stream (->> (.longs rng 0 (count vdata))
-                    (.iterator)
-                    (iterator-seq))]
-    (nth vdata (first stream))))
-
-(defn- collect
-  [f items]
-  (reduce #(conj %1 (f %2)) [] items))
-
-(defn- register-profile
-  [conn params]
-  (->> (#'profile/create-profile conn params)
-       (#'profile/create-profile-relations conn)))
-
-(defn impl-run
-  [pool opts]
-  (let [rng (java.util.Random. 1)]
-    (letfn [(create-profile [conn index]
-              (let [id   (mk-uuid "profile" index)
-                    _    (log/info "create profile" index id)
-
-                    prof (register-profile conn
-                                           {:id id
-                                            :fullname (str "Profile " index)
-                                            :password "123123"
-                                            :demo? true
-                                            :email (str "profile" index "@example.com")})
-                    team-id  (:default-team-id prof)
-                    owner-id id]
-                (let [project-ids (collect (partial create-project conn team-id owner-id)
-                                           (range (:num-projects-per-team opts)))]
-                  (run! (partial create-files conn owner-id) project-ids))
-                prof))
-
-            (create-profiles [conn]
-              (log/info "create profiles")
-              (collect (partial create-profile conn)
-                       (range (:num-profiles opts))))
-
-            (create-team [conn index]
-              (let [id (mk-uuid "team" index)
-                    name (str "Team" index)]
-                (log/info "create team" index id)
-                (db/insert! conn :team {:id id
-                                        :name name})
-                id))
-
-            (create-teams [conn]
-              (log/info "create teams")
-              (collect (partial create-team conn)
-                       (range (:num-teams opts))))
-
-            (create-file [conn owner-id project-id index]
-              (let [id (mk-uuid "file" project-id index)
-                    name (str "file" index)
-                    data (cp/make-file-data id)]
-                (log/info "create file" index id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-files [conn owner-id project-id]
-              (log/info "create files")
-              (run! (partial create-file conn owner-id project-id)
-                    (range (:num-files-per-project opts))))
-
-            (create-project [conn team-id owner-id index]
-              (let [id (mk-uuid "project" team-id index)
-                    name (str "project " index)]
-                (log/info "create project" index id)
-                (db/insert! conn :project
-                            {:id id
-                             :team-id team-id
-                             :name name})
-                (db/insert! conn :project-profile-rel
-                            {:project-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-projects [conn team-id profile-ids]
-              (log/info "create projects")
-              (let [owner-id (rng-nth rng profile-ids)
-                    project-ids (collect (partial create-project conn team-id owner-id)
-                                         (range (:num-projects-per-team opts)))]
-                (run! (partial create-files conn owner-id) project-ids)))
-
-            (assign-profile-to-team [conn team-id owner? profile-id]
-              (db/insert! conn :team-profile-rel
-                          {:team-id team-id
-                           :profile-id profile-id
-                           :is-owner owner?
-                           :is-admin true
-                           :can-edit true}))
-
-            (setup-team [conn team-id profile-ids]
-              (log/info "setup team" team-id profile-ids)
-              (assign-profile-to-team conn team-id true (first profile-ids))
-              (run! (partial assign-profile-to-team conn team-id false)
-                    (rest profile-ids))
-              (create-projects conn team-id profile-ids))
-
-            (assign-teams-and-profiles [conn teams profiles]
-              (log/info "assign teams and profiles")
-              (loop [team-id (first teams)
-                     teams (rest teams)]
-                (when-not (nil? team-id)
-                  (let [n-profiles-team (:num-profiles-per-team opts)
-                        selected-profiles (rng-vec rng profiles n-profiles-team)]
-                    (setup-team conn team-id selected-profiles)
-                    (recur (first teams)
-                           (rest teams))))))
-
-            (create-draft-file [conn owner index]
-              (let [owner-id   (:id owner)
-                    id         (mk-uuid "file" "draft" owner-id index)
-                    name       (str "file" index)
-                    project-id (:default-project-id owner)
-                    data       (cp/make-file-data id)]
-
-                (log/info "create draft file" index id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-draft-files [conn profile]
-              (run! (partial create-draft-file conn profile)
-                    (range (:num-draft-files-per-profile opts))))
-            ]
-      (db/with-atomic [conn pool]
-        (let [profiles (create-profiles conn)
-              teams    (create-teams conn)]
-          (assign-teams-and-profiles conn teams (map :id profiles))
-          (run! (partial create-draft-files conn) profiles))))))
-
-(defn run-in-system
-  [system preset]
-  (let [pool   (:app.db/pool system)
-        preset (if (map? preset)
-                 preset
-                 (case preset
-                   (nil "small" :small) preset-small
-                   ;; "medium" preset-medium
-                   ;; "big" preset-big
-                   preset-small))]
-    (impl-run pool preset)))
-
-(defn run
-  [{:keys [preset] :or {preset :small}}]
-  (let [config (select-keys (main/build-system-config cfg/config)
-                            [:app.db/pool
-                             :app.telemetry/migrations
-                             :app.migrations/migrations
-                             :app.migrations/all
-                             :app.metrics/metrics])
-        _      (ig/load-namespaces config)
-        system (-> (ig/prep config)
-                   (ig/init))]
-    (try
-      (run-in-system system preset)
-      (catch Exception e
-        (log/errorf e "Unhandled exception."))
-      (finally
-        (ig/halt! system)))))

backend/src/app/cli/manage.clj

@@ -0,0 +1,168 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.cli.manage
+  "A manage cli api."
+  (:require
+   [app.common.logging :as l]
+   [app.db :as db]
+   [app.main :as main]
+   [app.rpc.mutations.profile :as profile]
+   [app.rpc.queries.profile :refer [retrieve-profile-data-by-email]]
+   [clojure.string :as str]
+   [clojure.tools.cli :refer [parse-opts]]
+   [integrant.core :as ig])
+  (:import
+   java.io.Console))
+
+;; --- IMPL
+
+(defn init-system
+  []
+  (let [data (-> main/system-config
+                 (select-keys [:app.db/pool :app.metrics/metrics])
+                 (assoc :app.migrations/all {}))]
+    (-> data ig/prep ig/init)))
+
+(defn- read-from-console
+  [{:keys [label type] :or {type :text}}]
+  (let [^Console console (System/console)]
+    (when-not console
+      (l/error :hint "no console found, can proceed")
+      (System/exit 1))
+
+    (binding [*out* (.writer console)]
+      (print label " ")
+      (.flush *out*))
+
+    (case type
+      :text (.readLine console)
+      :password (String. (.readPassword console)))))
+
+(defn create-profile
+  [options]
+  (let [system   (init-system)
+        email    (or (:email options)
+                     (read-from-console {:label "Email:"}))
+        fullname (or (:fullname options)
+                     (read-from-console {:label "Full Name:"}))
+        password (or (:password options)
+                     (read-from-console {:label "Password:"
+                                         :type :password}))]
+    (try
+      (db/with-atomic [conn (:app.db/pool system)]
+        (->> (profile/create-profile conn
+                                     {:fullname fullname
+                                      :email email
+                                      :password password
+                                      :is-active true
+                                      :is-demo false})
+             (profile/create-profile-relations conn)))
+
+      (when (pos? (:verbosity options))
+        (println "User created successfully."))
+      (System/exit 0)
+
+      (catch Exception _e
+        (when (pos? (:verbosity options))
+          (println "Unable to create user, already exists."))
+        (System/exit 1)))))
+
+(defn reset-password
+  [options]
+  (let [system (init-system)]
+    (try
+      (db/with-atomic [conn (:app.db/pool system)]
+        (let [email    (or (:email options)
+                           (read-from-console {:label "Email:"}))
+              profile  (retrieve-profile-data-by-email conn email)]
+          (when-not profile
+            (when (pos? (:verbosity options))
+              (println "Profile does not exists."))
+            (System/exit 1))
+
+          (let [password (or (:password options)
+                             (read-from-console {:label "Password:"
+                                                 :type :password}))]
+            (profile/update-profile-password! conn (assoc profile :password password))
+            (when (pos? (:verbosity options))
+              (println "Password changed successfully.")))))
+      (System/exit 0)
+      (catch Exception e
+        (when (pos? (:verbosity options))
+          (println "Unable to change password."))
+        (when (= 2 (:verbosity options))
+          (.printStackTrace e))
+        (System/exit 1)))))
+
+;; --- CLI PARSE
+
+(def cli-options
+  ;; An option with a required argument
+  [["-u" "--email EMAIL" "Email Address"]
+   ["-p" "--password PASSWORD" "Password"]
+   ["-n" "--name FULLNAME" "Full Name"
+    :id :fullname]
+   ["-v" nil "Verbosity level"
+    :id :verbosity
+    :default 1
+    :update-fn inc]
+   ["-q" nil "Dont' print to console"
+    :id :verbosity
+    :update-fn (constantly 0)]
+   ["-h" "--help"]])
+
+(defn usage
+  [options-summary]
+  (->> ["Penpot CLI management."
+        ""
+        "Usage: manage [options] action"
+        ""
+        "Options:"
+        options-summary
+        ""
+        "Actions:"
+        "  create-profile    Create new profile."
+        "  reset-password    Reset profile password."
+        ""]
+       (str/join \newline)))
+
+(defn error-msg [errors]
+  (str "The following errors occurred while parsing your command:\n\n"
+       (str/join \newline errors)))
+
+(defn validate-args
+  "Validate command line arguments. Either return a map indicating the program
+  should exit (with a error message, and optional ok status), or a map
+  indicating the action the program should take and the options provided."
+  [args]
+  (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)]
+    (cond
+      (:help options) ; help => exit OK with usage summary
+      {:exit-message (usage summary) :ok? true}
+
+      errors ; errors => exit with description of errors
+      {:exit-message (error-msg errors)}
+
+      ;; custom validation on arguments
+      :else
+      (let [action (first arguments)]
+        (if (#{"create-profile" "reset-password"} action)
+          {:action (first arguments) :options options}
+          {:exit-message (usage summary)})))))
+
+(defn exit [status msg]
+  (println msg)
+  (System/exit status))
+
+(defn -main
+  [& args]
+  (let [{:keys [action options exit-message ok?]} (validate-args args)]
+    (if exit-message
+      (exit (if ok? 0 1) exit-message)
+      (case action
+        "create-profile" (create-profile options)
+        "reset-password" (reset-password options)))))

backend/src/app/cli/migrate_media.clj

@@ -2,19 +2,16 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.cli.migrate-media
   (:require
+   [app.common.logging :as l]
    [app.common.media :as cm]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.main :as main]
    [app.storage :as sto]
-   [clojure.tools.logging :as log]
    [cuerdas.core :as str]
    [datoteka.core :as fs]
    [integrant.core :as ig]))
@@ -34,7 +31,7 @@
 
 (defn run
   []
-  (let [config (select-keys (main/build-system-config cfg/config)
+  (let [config (select-keys main/system-config
                             [:app.db/pool
                              :app.migrations/migrations
                              :app.metrics/metrics
@@ -49,7 +46,7 @@
           (run-in-system)
           (ig/halt!))
       (catch Exception e
-        (log/errorf e "Unhandled exception.")))))
+        (l/error :hint "unhandled exception" :cause e)))))
 
 
 ;; --- IMPL
@@ -60,7 +57,7 @@
             (->> (db/exec! conn ["select * from profile"])
                  (filter #(not (str/empty? (:photo %))))
                  (seq)))]
-    (let [base    (fs/path (:storage-fs-old-directory cfg/config))
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
           storage (-> (:app.storage/storage system)
                       (assoc :conn conn))]
       (doseq [profile (retrieve-profiles conn)]
@@ -81,7 +78,7 @@
             (->> (db/exec! conn ["select * from team"])
                  (filter #(not (str/empty? (:photo %))))
                  (seq)))]
-    (let [base    (fs/path (:storage-fs-old-directory cfg/config))
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
           storage (-> (:app.storage/storage system)
                       (assoc :conn conn))]
       (doseq [team (retrieve-teams conn)]
@@ -105,7 +102,7 @@
                                     from file_media_object as fmo
                                     join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"])
                  (seq)))]
-    (let [base    (fs/path (:storage-fs-old-directory cfg/config))
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
           storage (-> (:app.storage/storage system)
                       (assoc :conn conn))]
       (doseq [mobj (retrieve-media-objects conn)]

backend/src/app/config.clj

@@ -2,154 +2,210 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.config
   "A configuration management."
+  (:refer-clojure :exclude [get])
   (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.flags :as flags]
    [app.common.spec :as us]
    [app.common.version :as v]
    [app.util.time :as dt]
+   [clojure.core :as c]
+   [clojure.java.io :as io]
+   [clojure.pprint :as pprint]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [environ.core :refer [env]]))
+   [environ.core :refer [env]]
+   [integrant.core :as ig]))
+
+(prefer-method print-method
+               clojure.lang.IRecord
+               clojure.lang.IDeref)
+
+(prefer-method pprint/simple-dispatch
+               clojure.lang.IPersistentMap
+               clojure.lang.IDeref)
+
+(defmethod ig/init-key :default
+  [_ data]
+  (d/without-nils data))
+
+(defmethod ig/prep-key :default
+  [_ data]
+  (if (map? data)
+    (d/without-nils data)
+    data))
 
 (def defaults
   {:http-server-port 6060
-
-   :database-uri "postgresql://127.0.0.1/penpot"
+   :http-server-host "0.0.0.0"
+   :host "devenv"
+   :tenant "dev"
+   :database-uri "postgresql://postgres/penpot"
    :database-username "penpot"
    :database-password "penpot"
 
-   :asserts-enabled false
+   :default-blob-version 3
+   :loggers-zmq-uri "tcp://localhost:45556"
+
+   :file-change-snapshot-every 5
+   :file-change-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
-   :redis-uri "redis://localhost/0"
+   :redis-uri "redis://redis/0"
 
    :srepl-host "127.0.0.1"
    :srepl-port 6062
 
-   :storage-backend :fs
-
-   :storage-fs-directory "resources/public/assets"
-   :storage-s3-region :eu-central-1
-   :storage-s3-bucket "penpot-devenv-assets-pre"
+   :assets-storage-backend :assets-fs
+   :storage-assets-fs-directory "assets"
 
    :assets-path "/internal/assets/"
 
-   :rlimits-password 10
-   :rlimits-image 2
+   :rlimit-password 10
+   :rlimit-image 2
+   :rlimit-font 5
 
-   :smtp-enabled false
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
-   :allow-demo-users true
-   :registration-enabled true
-   :registration-domain-whitelist ""
+   :profile-complaint-max-age (dt/duration {:days 7})
+   :profile-complaint-threshold 2
+
+   :profile-bounce-max-age (dt/duration {:days 7})
+   :profile-bounce-threshold 10
 
-   :telemetry-enabled false
-   :telemetry-with-taiga true
    :telemetry-uri "https://telemetry.penpot.app/"
 
-   ;; LDAP auth disabled by default. Set ldap-auth-host to enable
-   ;:ldap-auth-host "ldap.mysupercompany.com"
-   ;:ldap-auth-port 389
-   ;:ldap-bind-dn "cn=admin,dc=ldap,dc=mysupercompany,dc=com"
-   ;:ldap-bind-password "verysecure"
-   ;:ldap-auth-ssl false
-   ;:ldap-auth-starttls false
-   ;:ldap-auth-base-dn "ou=People,dc=ldap,dc=mysupercompany,dc=com"
+   :ldap-user-query "(|(uid=:username)(mail=:username))"
+   :ldap-attrs-username "uid"
+   :ldap-attrs-email "mail"
+   :ldap-attrs-fullname "cn"
+   :ldap-attrs-photo "jpegPhoto"
 
-   :ldap-auth-user-query "(|(uid=$username)(mail=$username))"
-   :ldap-auth-username-attribute "uid"
-   :ldap-auth-email-attribute "mail"
-   :ldap-auth-fullname-attribute "displayName"
-   :ldap-auth-avatar-attribute "jpegPhoto"
+   ;; a server prop key where initial project is stored.
+   :initial-project-skey "initial-project"})
 
-   ;; :initial-data-file "resources/initial-data.json"
-   ;; :initial-data-project-name "Penpot Oboarding"
-   })
+(s/def ::flags ::us/set-of-keywords)
 
-(s/def ::http-server-port ::us/integer)
-(s/def ::database-username (s/nilable ::us/string))
+;; DEPRECATED PROPERTIES: should be removed in 1.10
+(s/def ::registration-enabled ::us/boolean)
+(s/def ::smtp-enabled ::us/boolean)
+(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::asserts-enabled ::us/boolean)
+;; END DEPRECATED
+
+(s/def ::audit-log-archive-uri ::us/string)
+(s/def ::audit-log-gc-max-age ::dt/duration)
+
+(s/def ::admins ::us/set-of-str)
+(s/def ::file-change-snapshot-every ::us/integer)
+(s/def ::file-change-snapshot-timeout ::dt/duration)
+
+(s/def ::secret-key ::us/string)
+(s/def ::allow-demo-users ::us/boolean)
+(s/def ::assets-path ::us/string)
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
-(s/def ::redis-uri ::us/string)
-
-(s/def ::storage-backend ::us/keyword)
-(s/def ::storage-fs-directory ::us/string)
-(s/def ::assets-path ::us/string)
-(s/def ::storage-s3-region ::us/keyword)
-(s/def ::storage-s3-bucket ::us/string)
-
-(s/def ::media-uri ::us/string)
-(s/def ::media-directory ::us/string)
-(s/def ::asserts-enabled ::us/boolean)
-
+(s/def ::database-username (s/nilable ::us/string))
+(s/def ::default-blob-version ::us/integer)
 (s/def ::error-report-webhook ::us/string)
-(s/def ::smtp-enabled ::us/boolean)
-(s/def ::smtp-default-reply-to ::us/string)
-(s/def ::smtp-default-from ::us/string)
-(s/def ::smtp-host ::us/string)
-(s/def ::smtp-port ::us/integer)
-(s/def ::smtp-username (s/nilable ::us/string))
-(s/def ::smtp-password (s/nilable ::us/string))
-(s/def ::smtp-tls ::us/boolean)
-(s/def ::smtp-ssl ::us/boolean)
-(s/def ::allow-demo-users ::us/boolean)
-(s/def ::registration-enabled ::us/boolean)
-(s/def ::registration-domain-whitelist ::us/string)
-(s/def ::public-uri ::us/string)
-
-(s/def ::srepl-host ::us/string)
-(s/def ::srepl-port ::us/integer)
-
-(s/def ::rlimits-password ::us/integer)
-(s/def ::rlimits-image ::us/integer)
-
-(s/def ::google-client-id ::us/string)
-(s/def ::google-client-secret ::us/string)
-
-(s/def ::gitlab-client-id ::us/string)
-(s/def ::gitlab-client-secret ::us/string)
-(s/def ::gitlab-base-uri ::us/string)
-
+(s/def ::user-feedback-destination ::us/string)
 (s/def ::github-client-id ::us/string)
 (s/def ::github-client-secret ::us/string)
-
-(s/def ::ldap-auth-host ::us/string)
-(s/def ::ldap-auth-port ::us/integer)
+(s/def ::gitlab-base-uri ::us/string)
+(s/def ::gitlab-client-id ::us/string)
+(s/def ::gitlab-client-secret ::us/string)
+(s/def ::google-client-id ::us/string)
+(s/def ::google-client-secret ::us/string)
+(s/def ::oidc-client-id ::us/string)
+(s/def ::oidc-client-secret ::us/string)
+(s/def ::oidc-base-uri ::us/string)
+(s/def ::oidc-token-uri ::us/string)
+(s/def ::oidc-auth-uri ::us/string)
+(s/def ::oidc-user-uri ::us/string)
+(s/def ::oidc-scopes ::us/set-of-str)
+(s/def ::oidc-roles ::us/set-of-str)
+(s/def ::oidc-roles-attr ::us/keyword)
+(s/def ::host ::us/string)
+(s/def ::http-server-port ::us/integer)
+(s/def ::http-server-host ::us/string)
+(s/def ::http-session-idle-max-age ::dt/duration)
+(s/def ::http-session-updater-batch-max-age ::dt/duration)
+(s/def ::http-session-updater-batch-max-size ::us/integer)
+(s/def ::initial-project-skey ::us/string)
+(s/def ::ldap-attrs-email ::us/string)
+(s/def ::ldap-attrs-fullname ::us/string)
+(s/def ::ldap-attrs-photo ::us/string)
+(s/def ::ldap-attrs-username ::us/string)
+(s/def ::ldap-base-dn ::us/string)
 (s/def ::ldap-bind-dn ::us/string)
 (s/def ::ldap-bind-password ::us/string)
-(s/def ::ldap-auth-ssl ::us/boolean)
-(s/def ::ldap-auth-starttls ::us/boolean)
-(s/def ::ldap-auth-base-dn ::us/string)
-(s/def ::ldap-auth-user-query ::us/string)
-(s/def ::ldap-auth-username-attribute ::us/string)
-(s/def ::ldap-auth-email-attribute ::us/string)
-(s/def ::ldap-auth-fullname-attribute ::us/string)
-(s/def ::ldap-auth-avatar-attribute ::us/string)
-
-(s/def ::telemetry-enabled ::us/boolean)
-(s/def ::telemetry-with-taiga ::us/boolean)
+(s/def ::ldap-host ::us/string)
+(s/def ::ldap-port ::us/integer)
+(s/def ::ldap-ssl ::us/boolean)
+(s/def ::ldap-starttls ::us/boolean)
+(s/def ::ldap-user-query ::us/string)
+(s/def ::loggers-loki-uri ::us/string)
+(s/def ::loggers-zmq-uri ::us/string)
+(s/def ::media-directory ::us/string)
+(s/def ::media-uri ::us/string)
+(s/def ::profile-bounce-max-age ::dt/duration)
+(s/def ::profile-bounce-threshold ::us/integer)
+(s/def ::profile-complaint-max-age ::dt/duration)
+(s/def ::profile-complaint-threshold ::us/integer)
+(s/def ::public-uri ::us/string)
+(s/def ::redis-uri ::us/string)
+(s/def ::registration-domain-whitelist ::us/set-of-str)
+(s/def ::rlimit-font ::us/integer)
+(s/def ::rlimit-image ::us/integer)
+(s/def ::rlimit-password ::us/integer)
+(s/def ::smtp-default-from ::us/string)
+(s/def ::smtp-default-reply-to ::us/string)
+(s/def ::smtp-host ::us/string)
+(s/def ::smtp-password (s/nilable ::us/string))
+(s/def ::smtp-port ::us/integer)
+(s/def ::smtp-ssl ::us/boolean)
+(s/def ::smtp-tls ::us/boolean)
+(s/def ::smtp-username (s/nilable ::us/string))
+(s/def ::srepl-host ::us/string)
+(s/def ::srepl-port ::us/integer)
+(s/def ::assets-storage-backend ::us/keyword)
+(s/def ::fdata-storage-backend ::us/keyword)
+(s/def ::storage-assets-fs-directory ::us/string)
+(s/def ::storage-assets-s3-bucket ::us/string)
+(s/def ::storage-assets-s3-region ::us/keyword)
+(s/def ::storage-fdata-s3-bucket ::us/string)
+(s/def ::storage-fdata-s3-region ::us/keyword)
+(s/def ::storage-fdata-s3-prefix ::us/string)
 (s/def ::telemetry-uri ::us/string)
-(s/def ::telemetry-server-enabled ::us/boolean)
-(s/def ::telemetry-server-port ::us/integer)
+(s/def ::telemetry-with-taiga ::us/boolean)
+(s/def ::tenant ::us/string)
 
-(s/def ::initial-data-file ::us/string)
-(s/def ::initial-data-project-name ::us/string)
+(s/def ::sentry-trace-sample-rate ::us/number)
+(s/def ::sentry-attach-stack-trace ::us/boolean)
+(s/def ::sentry-debug ::us/boolean)
+(s/def ::sentry-dsn ::us/string)
 
 (s/def ::config
-  (s/keys :opt-un [::allow-demo-users
-                   ::asserts-enabled
+  (s/keys :opt-un [::secret-key
+                   ::flags
+                   ::admins
+                   ::allow-demo-users
+                   ::audit-log-archive-uri
+                   ::audit-log-gc-max-age
                    ::database-password
                    ::database-uri
                    ::database-username
+                   ::default-blob-version
                    ::error-report-webhook
+                   ::file-change-snapshot-every
+                   ::file-change-snapshot-timeout
+                   ::user-feedback-destination
                    ::github-client-id
                    ::github-client-secret
                    ::gitlab-base-uri
@@ -157,25 +213,52 @@
                    ::gitlab-client-secret
                    ::google-client-id
                    ::google-client-secret
+                   ::oidc-client-id
+                   ::oidc-client-secret
+                   ::oidc-base-uri
+                   ::oidc-token-uri
+                   ::oidc-auth-uri
+                   ::oidc-user-uri
+                   ::oidc-scopes
+                   ::oidc-roles-attr
+                   ::oidc-roles
+                   ::host
+                   ::http-server-host
                    ::http-server-port
-                   ::ldap-auth-avatar-attribute
-                   ::ldap-auth-base-dn
-                   ::ldap-auth-email-attribute
-                   ::ldap-auth-fullname-attribute
-                   ::ldap-auth-host
-                   ::ldap-auth-port
-                   ::ldap-auth-ssl
-                   ::ldap-auth-starttls
-                   ::ldap-auth-user-query
-                   ::ldap-auth-username-attribute
+                   ::http-session-idle-max-age
+                   ::http-session-updater-batch-max-age
+                   ::http-session-updater-batch-max-size
+                   ::initial-project-skey
+                   ::ldap-attrs-email
+                   ::ldap-attrs-fullname
+                   ::ldap-attrs-photo
+                   ::ldap-attrs-username
+                   ::ldap-base-dn
                    ::ldap-bind-dn
                    ::ldap-bind-password
+                   ::ldap-host
+                   ::ldap-port
+                   ::ldap-ssl
+                   ::ldap-starttls
+                   ::ldap-user-query
+                   ::local-assets-uri
+                   ::loggers-loki-uri
+                   ::loggers-zmq-uri
+                   ::profile-bounce-max-age
+                   ::profile-bounce-threshold
+                   ::profile-complaint-max-age
+                   ::profile-complaint-threshold
                    ::public-uri
                    ::redis-uri
                    ::registration-domain-whitelist
                    ::registration-enabled
-                   ::rlimits-password
-                   ::rlimits-image
+                   ::rlimit-font
+                   ::rlimit-image
+                   ::rlimit-password
+                   ::sentry-dsn
+                   ::sentry-debug
+                   ::sentry-attach-stack-trace
+                   ::sentry-trace-sample-rate
                    ::smtp-default-from
                    ::smtp-default-reply-to
                    ::smtp-enabled
@@ -185,51 +268,77 @@
                    ::smtp-ssl
                    ::smtp-tls
                    ::smtp-username
-                   ::storage-backend
-                   ::storage-fs-directory
                    ::srepl-host
                    ::srepl-port
-                   ::local-assets-uri
-                   ::storage-s3-bucket
-                   ::storage-s3-region
+                   ::assets-storage-backend
+                   ::storage-assets-fs-directory
+                   ::storage-assets-s3-bucket
+                   ::storage-assets-s3-region
+                   ::fdata-storage-backend
+                   ::storage-fdata-s3-bucket
+                   ::storage-fdata-s3-region
+                   ::storage-fdata-s3-prefix
                    ::telemetry-enabled
-                   ::telemetry-with-taiga
-                   ::telemetry-server-enabled
-                   ::telemetry-server-port
                    ::telemetry-uri
-                   ::initial-data-file
-                   ::initial-data-project-name]))
+                   ::telemetry-referer
+                   ::telemetry-with-taiga
+                   ::tenant]))
 
-(defn- env->config
-  [env]
-  (reduce-kv
-   (fn [acc k v]
-     (cond-> acc
-       (str/starts-with? (name k) "penpot-")
-       (assoc (keyword (subs (name k) 7)) v)
+(def default-flags
+  [:enable-backend-asserts
+   :enable-backend-api-doc
+   :enable-secure-session-cookies])
 
-       (str/starts-with? (name k) "app-")
-       (assoc (keyword (subs (name k) 4)) v)))
-   {}
-   env))
+(defn- parse-flags
+  [config]
+  (flags/parse flags/default
+               default-flags
+               (:flags config)))
+
+(defn read-env
+  [prefix]
+  (let [prefix (str prefix "-")
+        len    (count prefix)]
+    (reduce-kv
+     (fn [acc k v]
+       (cond-> acc
+         (str/starts-with? (name k) prefix)
+         (assoc (keyword (subs (name k) len)) v)))
+     {}
+     env)))
 
 (defn- read-config
-  [env]
-  (->> (env->config env)
-       (merge defaults)
-       (us/conform ::config)))
+  []
+  (try
+    (->> (read-env "penpot")
+         (merge defaults)
+         (us/conform ::config))
+    (catch Throwable e
+      (when (ex/ex-info? e)
+        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")
+        (println "Error on validating configuration:")
+        (println (:explain (ex-data e))
+        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
+      (throw e))))
 
-(defn- read-test-config
-  [env]
-  (merge {:redis-uri "redis://redis/1"
-          :database-uri "postgresql://postgres/penpot_test"
-          :storage-fs-directory "/tmp/app/storage"
-          :migrations-verbose false}
-         (read-config env)))
+(def version
+  (v/parse (or (some-> (io/resource "version.txt")
+                       (slurp)
+                       (str/trim))
+               "%version%")))
 
-(def version (v/parse "%version%"))
-(def config (read-config env))
-(def test-config (read-test-config env))
+(def ^:dynamic config (read-config))
+(def ^:dynamic flags  (parse-flags config))
 
-(def default-deletion-delay
-  (dt/duration {:hours 48}))
+(def deletion-delay
+  (dt/duration {:days 7}))
+
+(defn get
+  "A configuration getter. Helps code be more testable."
+  ([key]
+   (c/get config key))
+  ([key default]
+   (c/get config key default)))
+
+;; Set value for all new threads bindings.
+(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -2,21 +2,22 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.db
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
+   [app.common.logging :as l]
    [app.common.spec :as us]
+   [app.common.transit :as t]
+   [app.common.uuid :as uuid]
    [app.db.sql :as sql]
+   [app.metrics :as mtx]
    [app.util.json :as json]
    [app.util.migrations :as mg]
    [app.util.time :as dt]
-   [app.util.transit :as t]
    [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
@@ -26,13 +27,16 @@
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
+   java.io.InputStream
+   java.io.OutputStream
+   java.lang.AutoCloseable
    java.sql.Connection
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
+   org.postgresql.jdbc.PgArray
    org.postgresql.largeobject.LargeObject
    org.postgresql.largeobject.LargeObjectManager
-   org.postgresql.jdbc.PgArray
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
@@ -43,61 +47,97 @@
 ;; Initialization
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(s/def ::uri ::us/not-empty-string)
-(s/def ::name ::us/not-empty-string)
-(s/def ::min-pool-size ::us/integer)
+(declare instrument-jdbc!)
+(declare apply-migrations!)
+
+(s/def ::connection-timeout ::us/integer)
 (s/def ::max-pool-size ::us/integer)
 (s/def ::migrations map?)
-(s/def ::metrics map?)
+(s/def ::min-pool-size ::us/integer)
+(s/def ::name keyword?)
+(s/def ::password ::us/string)
+(s/def ::read-only ::us/boolean)
+(s/def ::uri ::us/not-empty-string)
+(s/def ::username ::us/string)
+(s/def ::validation-timeout ::us/integer)
 
 (defmethod ig/pre-init-spec ::pool [_]
-  (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size ::migrations]))
+  (s/keys :req-un [::uri ::name ::username ::password]
+          :opt-un [::min-pool-size
+                   ::max-pool-size
+                   ::connection-timeout
+                   ::validation-timeout
+                   ::migrations
+                   ::mtx/metrics
+                   ::read-only]))
 
 (defmethod ig/init-key ::pool
-  [_ {:keys [migrations] :as cfg}]
+  [_ {:keys [migrations metrics name] :as cfg}]
+  (l/info :action "initialize connection pool" :name (d/name name) :uri (:uri cfg))
+  (some-> metrics :registry instrument-jdbc!)
+
   (let [pool (create-pool cfg)]
-    (when (seq migrations)
-      (with-open [conn (open pool)]
-        (mg/setup! conn)
-        (doseq [[mname steps] migrations]
-          (mg/migrate! conn {:name (name mname) :steps steps}))))
+    (some->> (seq migrations) (apply-migrations! pool))
     pool))
 
 (defmethod ig/halt-key! ::pool
   [_ pool]
   (.close ^HikariDataSource pool))
 
+(defn- instrument-jdbc!
+  [registry]
+  (mtx/instrument-vars!
+   [#'next.jdbc/execute-one!
+    #'next.jdbc/execute!]
+   {:registry registry
+    :type :counter
+    :name "database_query_total"
+    :help "An absolute counter of database queries."}))
+
+(defn- apply-migrations!
+  [pool migrations]
+  (with-open [conn ^AutoCloseable (open pool)]
+    (mg/setup! conn)
+    (doseq [[name steps] migrations]
+      (mg/migrate! conn {:name (d/name name) :steps steps}))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; API & Impl
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def initsql
-  (str "SET statement_timeout = 10000;\n"
-       "SET idle_in_transaction_session_timeout = 120000;"))
+  (str "SET statement_timeout = 300000;\n"
+       "SET idle_in_transaction_session_timeout = 300000;"))
 
 (defn- create-datasource-config
-  [{:keys [metrics] :as cfg}]
+  [{:keys [metrics read-only] :or {read-only false} :as cfg}]
   (let [dburi    (:uri cfg)
         username (:username cfg)
         password (:password cfg)
-        config   (HikariConfig.)
-        mtf      (PrometheusMetricsTrackerFactory. (:registry metrics))]
+        config   (HikariConfig.)]
     (doto config
       (.setJdbcUrl (str "jdbc:" dburi))
-      (.setPoolName (:name cfg "default"))
+      (.setPoolName (d/name (:name cfg)))
       (.setAutoCommit true)
-      (.setReadOnly false)
-      (.setConnectionTimeout 8000)  ;; 8seg
-      (.setValidationTimeout 8000)  ;; 8seg
-      (.setIdleTimeout 120000)      ;; 2min
-      (.setMaxLifetime 1800000)     ;; 30min
-      (.setMinimumIdle (:min-pool-size cfg 0))
-      (.setMaximumPoolSize (:max-pool-size cfg 30))
-      (.setMetricsTrackerFactory mtf)
+      (.setReadOnly read-only)
+      (.setConnectionTimeout (:connection-timeout cfg 10000))  ;; 10seg
+      (.setValidationTimeout (:validation-timeout cfg 10000))  ;; 10seg
+      (.setIdleTimeout 120000)       ;; 2min
+      (.setMaxLifetime 1800000)      ;; 30min
+      (.setMinimumIdle     (:min-pool-size cfg 0))
+      (.setMaximumPoolSize (:max-pool-size cfg 50))
       (.setConnectionInitSql initsql)
       (.setInitializationFailTimeout -1))
+
+    ;; When metrics namespace is provided
+    (when metrics
+      (->> (:registry metrics)
+           (PrometheusMetricsTrackerFactory.)
+           (.setMetricsTrackerFactory config)))
+
     (when username (.setUsername config username))
     (when password (.setPassword config password))
+
     config))
 
 (defn pool?
@@ -110,7 +150,7 @@
   [pool]
   (.isClosed ^HikariDataSource pool))
 
-(defn- create-pool
+(defn create-pool
   [cfg]
   (let [dsc (create-datasource-config cfg)]
     (jdbc-dt/read-as-instant)
@@ -162,7 +202,7 @@
   [& args]
   `(jdbc/with-transaction ~@args))
 
-(defn open
+(defn ^Connection open
   [pool]
   (jdbc/get-connection pool))
 
@@ -185,9 +225,11 @@
               (assoc opts :return-keys true))))
 
 (defn insert-multi!
-  [ds table param-list]
-  (doseq [params param-list]
-    (insert! ds table params)))
+  ([ds table cols rows] (insert-multi! ds table cols rows nil))
+  ([ds table cols rows opts]
+   (exec! ds
+          (sql/insert-multi table cols rows opts)
+          (assoc opts :return-keys true))))
 
 (defn update!
   ([ds table params where] (update! ds table params where nil))
@@ -203,13 +245,20 @@
               (sql/delete table params opts)
               (assoc opts :return-keys true))))
 
+(defn- is-deleted?
+  [{:keys [deleted-at]}]
+  (and (dt/instant? deleted-at)
+       (< (inst-ms deleted-at)
+          (inst-ms (dt/now)))))
+
 (defn get-by-params
   ([ds table params]
    (get-by-params ds table params nil))
-  ([ds table params opts]
+  ([ds table params {:keys [check-not-found] :or {check-not-found true} :as opts}]
    (let [res (exec-one! ds (sql/select table params opts))]
-     (when (or (:deleted-at res) (not res))
+     (when (and check-not-found (or (not res) (is-deleted? res)))
        (ex/raise :type :not-found
+                 :table table
                  :hint "database object not found"))
      res)))
 
@@ -226,8 +275,11 @@
    (exec! ds (sql/select table params opts))))
 
 (defn pgobject?
-  [v]
-  (instance? PGobject v))
+  ([v]
+   (instance? PGobject v))
+  ([v type]
+   (and (instance? PGobject v)
+        (= type (.getType ^PGobject v)))))
 
 (defn pginterval?
   [v]
@@ -238,21 +290,38 @@
   (instance? PGpoint v))
 
 (defn pgarray?
-  [v]
-  (instance? PgArray v))
+  ([v] (instance? PgArray v))
+  ([v type]
+   (and (instance? PgArray v)
+        (= type (.getBaseTypeName ^PgArray v)))))
 
 (defn pgarray-of-uuid?
   [v]
   (and (pgarray? v) (= "uuid" (.getBaseTypeName ^PgArray v))))
 
+(defn decode-pgarray
+  ([v] (into [] (.getArray ^PgArray v)))
+  ([v in] (into in (.getArray ^PgArray v)))
+  ([v in xf] (into in xf (.getArray ^PgArray v))))
+
+(defn pgarray->set
+  [v]
+  (set (.getArray ^PgArray v)))
+
+(defn pgarray->vector
+  [v]
+  (vec (.getArray ^PgArray v)))
+
 (defn pgpoint
   [p]
   (PGpoint. (:x p) (:y p)))
 
 (defn create-array
-  [conn type aobjects]
+  [conn type objects]
   (let [^PGConnection conn (unwrap conn org.postgresql.PGConnection)]
-    (.createArrayOf conn ^String type aobjects)))
+    (if (coll? objects)
+      (.createArrayOf conn ^String type (into-array Object objects))
+      (.createArrayOf conn ^String type objects))))
 
 (defn decode-pgpoint
   [^PGpoint v]
@@ -286,7 +355,7 @@
     (pginterval data)
 
     (dt/duration? data)
-    (->> (/ (.toMillis data) 1000.0)
+    (->> (/ (.toMillis ^java.time.Duration data) 1000.0)
          (format "%s seconds")
          (pginterval))
 
@@ -299,7 +368,7 @@
         val (.getValue o)]
     (if (or (= typ "json")
             (= typ "jsonb"))
-      (json/decode-str val)
+      (json/read val)
       val)))
 
 (defn decode-transit-pgobject
@@ -311,24 +380,49 @@
       (t/decode-str val)
       val)))
 
+(defn inet
+  [ip-addr]
+  (doto (org.postgresql.util.PGobject.)
+    (.setType "inet")
+    (.setValue (str ip-addr))))
+
+(defn decode-inet
+  [^PGobject o]
+  (if (= "inet" (.getType o))
+    (.getValue o)
+    nil))
+
 (defn tjson
   "Encode as transit json."
   [data]
   (doto (org.postgresql.util.PGobject.)
     (.setType "jsonb")
-    (.setValue (t/encode-verbose-str data))))
+    (.setValue (t/encode-str data {:type :json-verbose}))))
 
 (defn json
   "Encode as plain json."
   [data]
   (doto (org.postgresql.util.PGobject.)
     (.setType "jsonb")
-    (.setValue (json/encode-str data))))
+    (.setValue (json/write-str data))))
 
-(defn pgarray->set
-  [v]
-  (set (.getArray ^PgArray v)))
+;; --- Locks
 
-(defn pgarray->vector
-  [v]
-  (vec (.getArray ^PgArray v)))
+(defn- xact-check-param
+  [n]
+  (cond
+    (uuid? n) (uuid/get-word-high n)
+    (int? n)  n
+    :else (throw (IllegalArgumentException. "uuid or number allowed"))))
+
+(defn xact-lock!
+  [conn n]
+  (let [n (xact-check-param n)]
+    (exec-one! conn ["select pg_advisory_xact_lock(?::bigint) as lock" n])
+    true))
+
+(defn xact-try-lock!
+  [conn n]
+  (let [n   (xact-check-param n)
+        row (exec-one! conn ["select pg_try_advisory_xact_lock(?::bigint) as lock" n])]
+    (:lock row)))

backend/src/app/db/profile_initial_data.clj

@@ -1,117 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
-
-(ns app.db.profile-initial-data
-  (:require
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.rpc.mutations.projects :as projects]
-   [app.util.transit :as tr]
-   [clojure.java.io :as io]
-   [datoteka.core :as fs]))
-
-(def sql:file
-  "select * from file where project_id = ?")
-
-(def sql:file-library-rel
-  "with file_ids as (select id from file where project_id = ?)
-   select *
-     from file_library_rel
-     where file_id in (select id from file_ids)")
-
-(def sql:file-media-object
-  "with file_ids as (select id from file where project_id = ?)
-   select *
-     from file_media_object
-     where file_id in (select id from file_ids)")
-
-(defn change-ids
-  "Given a collection and a map from ID to ID. Changes all the `keys` properties
-  so they point to the new ID existing in `map-ids`"
-  [map-ids coll keys]
-  (let [generate-id
-        (fn [map-ids {:keys [id]}]
-          (assoc map-ids id (uuid/next)))
-
-        remap-key
-        (fn [obj map-ids key]
-          (cond-> obj
-            (contains? obj key)
-            (assoc key (get map-ids (get obj key) (get obj key)))))
-
-        change-id
-        (fn [map-ids obj]
-          (reduce #(remap-key %1 map-ids %2) obj keys))
-
-        new-map-ids (reduce generate-id map-ids coll)]
-
-    [new-map-ids (map (partial change-id new-map-ids) coll)]))
-
-(defn create-initial-data-dump
-  [conn project-id output-path]
-  (let [ ;; Retrieve data from templates
-        opath                (fs/path output-path)
-        file                 (db/exec! conn [sql:file, project-id])
-        file-library-rel     (db/exec! conn [sql:file-library-rel, project-id])
-        file-media-object    (db/exec! conn [sql:file-media-object, project-id])
-
-        data {:file file
-              :file-library-rel file-library-rel
-              :file-media-object file-media-object}]
-    (with-open [output (io/output-stream opath)]
-      (tr/encode-stream data output)
-      nil)))
-
-(defn read-initial-data
-  [path]
-  (when (fs/exists? path)
-    (with-open [input (io/input-stream (fs/path path))]
-      (tr/decode-stream input))))
-
-(defn create-profile-initial-data
-  ([conn profile]
-   (when-let [initial-data-path (:initial-data-file cfg/config)]
-     (create-profile-initial-data conn initial-data-path profile)))
-
-  ([conn file profile]
-   (when-let [{:keys [file file-library-rel file-media-object]} (read-initial-data file)]
-     (let [sample-project-name (:initial-data-project-name cfg/config "Penpot Onboarding")
-
-           proj (projects/create-project conn {:profile-id (:id profile)
-                                               :team-id (:default-team-id profile)
-                                               :name sample-project-name})
-
-           map-ids {}
-
-           ;; Create new ID's and change the references
-           [map-ids file]                 (change-ids map-ids file #{:id})
-           [map-ids file-library-rel]     (change-ids map-ids file-library-rel #{:file-id :library-file-id})
-           [_       file-media-object]    (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id})
-
-           file             (map #(assoc % :project-id (:id proj)) file)
-           file-profile-rel (map #(array-map :file-id (:id %)
-                                             :profile-id (:id profile)
-                                             :is-owner true
-                                             :is-admin true
-                                             :can-edit true)
-                                 file)]
-
-       (projects/create-project-profile conn {:project-id (:id proj)
-                                              :profile-id (:id profile)})
-
-       (projects/create-team-project-profile conn {:team-id (:default-team-id profile)
-                                                   :project-id (:id proj)
-                                                   :profile-id (:id profile)})
-
-       ;; Re-insert into the database
-       (db/insert-multi! conn :file file)
-       (db/insert-multi! conn :file-profile-rel file-profile-rel)
-       (db/insert-multi! conn :file-library-rel file-library-rel)
-       (db/insert-multi! conn :file-media-object file-media-object)))))

backend/src/app/db/sql.clj

@@ -2,10 +2,7 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.db.sql
   (:refer-clojure :exclude [update])
@@ -35,14 +32,19 @@
                 (assoc :suffix "ON CONFLICT DO NOTHING"))]
      (sql/for-insert table key-map opts))))
 
+(defn insert-multi
+  [table cols rows opts]
+  (let [opts (merge default-opts opts)]
+    (sql/for-insert-multi table cols rows opts)))
+
 (defn select
   ([table where-params]
    (select table where-params nil))
   ([table where-params opts]
    (let [opts (merge default-opts opts)
          opts (cond-> opts
-                (:for-update opts)
-                (assoc :suffix "FOR UPDATE"))]
+                (:for-update opts)    (assoc :suffix "FOR UPDATE")
+                (:for-key-share opts) (assoc :suffix "FOR KEY SHARE"))]
      (sql/for-query table where-params opts))))
 
 (defn update
@@ -58,4 +60,3 @@
   ([table where-params opts]
    (let [opts (merge default-opts opts)]
      (sql/for-delete table where-params opts))))
-

backend/src/app/emails.clj

@@ -2,28 +2,22 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.emails
   "Main api for send emails."
   (:require
+   [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.config :as cfg]
-   [app.tasks :as tasks]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.db.sql :as sql]
    [app.util.emails :as emails]
-   [clojure.spec.alpha :as s]))
+   [app.worker :as wrk]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
 
-;; --- Defaults
-
-(defn default-context
-  []
-  {:assets-uri (:assets-uri cfg/config)
-   :public-uri (:public-uri cfg/config)})
-
-;; --- Public API
+;; --- PUBLIC API
 
 (defn render
   [email-factory context]
@@ -31,17 +25,78 @@
 
 (defn send!
   "Schedule the email for sending."
-  [conn email-factory context]
-  (us/verify fn? email-factory)
-  (us/verify map? context)
-  (let [email (email-factory context)]
-    (tasks/submit! conn {:name "sendmail"
-                         :delay 0
-                         :max-retries 1
-                         :priority 200
-                         :props email})))
+  [{:keys [::conn ::factory] :as context}]
+  (us/verify fn? factory)
+  (us/verify some? conn)
+  (let [email (factory context)]
+    (wrk/submit! (assoc email
+                        ::wrk/task :sendmail
+                        ::wrk/delay 0
+                        ::wrk/max-retries 1
+                        ::wrk/priority 200
+                        ::wrk/conn conn))))
 
-;; --- Emails
+
+;; --- BOUNCE/COMPLAINS HANDLING
+
+(def sql:profile-complaint-report
+  "select (select count(*)
+             from profile_complaint_report
+            where type = 'complaint'
+              and profile_id = ?
+              and created_at > now() - ?::interval) as complaints,
+          (select count(*)
+             from profile_complaint_report
+            where type = 'bounce'
+              and profile_id = ?
+              and created_at > now() - ?::interval) as bounces;")
+
+(defn allow-send-emails?
+  [conn profile]
+  (when-not (:is-muted profile false)
+    (let [complaint-threshold (cf/get :profile-complaint-threshold)
+          complaint-max-age   (cf/get :profile-complaint-max-age)
+          bounce-threshold    (cf/get :profile-bounce-threshold)
+          bounce-max-age      (cf/get :profile-bounce-max-age)
+
+          {:keys [complaints bounces] :as result}
+          (db/exec-one! conn [sql:profile-complaint-report
+                              (:id profile)
+                              (db/interval complaint-max-age)
+                              (:id profile)
+                              (db/interval bounce-max-age)])]
+
+      (and (< (or complaints 0) complaint-threshold)
+           (< (or bounces 0) bounce-threshold)))))
+
+(defn has-complaint-reports?
+  ([conn email] (has-complaint-reports? conn email nil))
+  ([conn email {:keys [threshold] :or {threshold 1}}]
+   (let [reports (db/exec! conn (sql/select :global-complaint-report
+                                            {:email email :type "complaint"}
+                                            {:limit 10}))]
+     (>= (count reports) threshold))))
+
+(defn has-bounce-reports?
+  ([conn email] (has-bounce-reports? conn email nil))
+  ([conn email {:keys [threshold] :or {threshold 1}}]
+   (let [reports (db/exec! conn (sql/select :global-complaint-report
+                                            {:email email :type "bounce"}
+                                            {:limit 10}))]
+     (>= (count reports) threshold))))
+
+
+;; --- EMAIL FACTORIES
+
+(s/def ::subject ::us/string)
+(s/def ::content ::us/string)
+
+(s/def ::feedback
+  (s/keys :req-un [::subject ::content]))
+
+(def feedback
+  "A profile feedback email."
+  (emails/template-factory ::feedback))
 
 (s/def ::name ::us/string)
 (s/def ::register
@@ -49,7 +104,7 @@
 
 (def register
   "A new profile registration welcome email."
-  (emails/template-factory ::register default-context))
+  (emails/template-factory ::register))
 
 (s/def ::token ::us/string)
 (s/def ::password-recovery
@@ -57,7 +112,7 @@
 
 (def password-recovery
   "A password recovery notification email."
-  (emails/template-factory ::password-recovery default-context))
+  (emails/template-factory ::password-recovery))
 
 (s/def ::pending-email ::us/email)
 (s/def ::change-email
@@ -65,17 +120,64 @@
 
 (def change-email
   "Password change confirmation email"
-  (emails/template-factory ::change-email default-context))
+  (emails/template-factory ::change-email))
 
 (s/def :internal.emails.invite-to-team/invited-by ::us/string)
 (s/def :internal.emails.invite-to-team/team ::us/string)
 (s/def :internal.emails.invite-to-team/token ::us/string)
 
 (s/def ::invite-to-team
-  (s/keys :keys [:internal.emails.invite-to-team/invited-by
-                 :internal.emails.invite-to-team/token
-                 :internal.emails.invite-to-team/team]))
+  (s/keys :req-un [:internal.emails.invite-to-team/invited-by
+                   :internal.emails.invite-to-team/token
+                   :internal.emails.invite-to-team/team]))
 
 (def invite-to-team
   "Teams member invitation email."
-  (emails/template-factory ::invite-to-team default-context))
+  (emails/template-factory ::invite-to-team))
+
+
+;; --- SENDMAIL TASK
+
+(declare send-console!)
+
+(s/def ::username ::cf/smtp-username)
+(s/def ::password ::cf/smtp-password)
+(s/def ::tls ::cf/smtp-tls)
+(s/def ::ssl ::cf/smtp-ssl)
+(s/def ::host ::cf/smtp-host)
+(s/def ::port ::cf/smtp-port)
+(s/def ::default-reply-to ::cf/smtp-default-reply-to)
+(s/def ::default-from ::cf/smtp-default-from)
+
+(defmethod ig/pre-init-spec ::sendmail-handler [_]
+  (s/keys :opt-un [::username
+                   ::password
+                   ::tls
+                   ::ssl
+                   ::host
+                   ::port
+                   ::default-from
+                   ::default-reply-to]))
+
+(defmethod ig/init-key ::sendmail-handler
+  [_ cfg]
+  (fn [{:keys [props] :as task}]
+    (let [enabled? (or (contains? cf/flags :smtp)
+                       (cf/get :smtp-enabled)
+                       (:enabled task))]
+      (if enabled?
+        (emails/send! cfg props)
+        (send-console! cfg props)))))
+
+(defn- send-console!
+  [cfg email]
+  (let [baos (java.io.ByteArrayOutputStream.)
+        mesg (emails/smtp-message cfg email)]
+    (.writeTo mesg baos)
+    (let [out (with-out-str
+                (println "email console dump:")
+                (println "******** start email" (:id email) "**********")
+                (println (.toString baos))
+                (println "******** end email "(:id email) "**********"))]
+      (l/info :email out))))
+

backend/src/app/error_reporter.clj

@@ -1,160 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
-
-(ns app.error-reporter
-  "A mattermost integration for error reporting."
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.util.http :as http]
-   [app.util.json :as json]
-   [app.util.template :as tmpl]
-   [app.worker :as wrk]
-   [clojure.core.async :as a]
-   [clojure.java.io :as io]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [cuerdas.core :as str]
-   [integrant.core :as ig]
-   [promesa.exec :as px])
-  (:import
-   org.apache.logging.log4j.core.LogEvent
-   org.apache.logging.log4j.util.ReadOnlyStringMap))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Error Listener
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(declare handle-event)
-
-(defonce enabled-mattermost (atom true))
-(defonce queue (a/chan (a/sliding-buffer 64)))
-(defonce queue-fn (fn [event] (a/>!! queue event)))
-
-(s/def ::uri ::us/string)
-
-(defmethod ig/pre-init-spec ::reporter [_]
-  (s/keys :req-un [::wrk/executor ::db/pool]
-          :opt-un [::uri]))
-
-(defmethod ig/init-key ::reporter
-  [_ {:keys [executor] :as cfg}]
-  (log/info "Intializing error reporter.")
-  (let [close-ch (a/chan 1)]
-    (a/go-loop []
-      (let [[val port] (a/alts! [close-ch queue])]
-        (cond
-          (= port close-ch)
-          (log/info "Stoping error reporting loop.")
-
-          (nil? val)
-          (log/info "Stoping error reporting loop.")
-
-          :else
-          (do
-            (px/run! executor #(handle-event cfg val))
-            (recur)))))
-    close-ch))
-
-(defmethod ig/halt-key! ::reporter
-  [_ close-ch]
-  (a/close! close-ch))
-
-(defn- get-context-data
-  [event]
-  (let [^LogEvent levent (deref event)
-        ^ReadOnlyStringMap rosm (.getContextData levent)]
-    (into {:message (str event)
-           :id      (uuid/next)} ; set default uuid for cases when it not comes.
-          (comp
-           (map (fn [[key val]]
-                  (cond
-                    (= "id" key)         [:id (uuid/uuid val)]
-                    (= "profile-id" key) [:profile-id (uuid/uuid val)]
-                    (str/blank? val)     nil
-                    (string? key)        [(keyword key) val]
-                    :else                [key val])))
-           (filter some?))
-
-          (.toMap rosm))))
-
-(defn- send-mattermost-notification!
-  [cfg {:keys [message host version id] :as cdata}]
-  (try
-    (let [uri    (:uri cfg)
-          prefix (str "Unhandled exception (@channel):\n"
-                      "- detail: " (:public-uri cfg/config) "/dbg/error-by-id/" id "\n"
-                      "- host: `" host "`\n"
-                      "- version: `" version "`\n")
-          text   (str prefix "```\n" message "\n```")
-          rsp    (http/send! {:uri uri
-                              :method :post
-                              :headers {"content-type" "application/json"}
-                              :body (json/encode-str {:text text})})]
-      (when (not= (:status rsp) 200)
-        (log/warnf "Error reporting webhook replying with unexpected status: %s\n%s"
-                   (:status rsp)
-                   (pr-str rsp))))
-
-    (catch Exception e
-      (log/warnf e "Unexpected exception on error reporter."))))
-
-(defn- persist-on-database!
-  [{:keys [pool] :as cfg} {:keys [id] :as cdata}]
-  (db/with-atomic [conn pool]
-    (db/insert! conn :server-error-report
-                {:id id :content (db/tjson cdata)})))
-
-(defn handle-event
-  [cfg event]
-  (try
-    (let [cdata (get-context-data event)]
-      (when (and (:uri cfg) @enabled-mattermost)
-        (send-mattermost-notification! cfg cdata))
-      (persist-on-database! cfg cdata))
-    (catch Exception e
-      (log/warnf e "Unexpected exception on error reporter."))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Handler
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as cfg}]
-  (letfn [(parse-id [request]
-            (let [id (get-in request [:path-params :id])
-                  id (us/uuid-conformer id)]
-              (when (uuid? id)
-                id)))
-          (retrieve-report [id]
-            (ex/ignoring
-             (when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
-               (assoc row :content (db/decode-transit-pgobject content)))))
-
-          (render-template [{:keys [content] :as report}]
-            (some-> (io/resource "error-report.tmpl")
-                    (tmpl/render content)))]
-
-
-    (fn [request]
-      (let [result (some-> (parse-id request)
-                           (retrieve-report)
-                           (render-template))]
-        (if result
-          {:status 200
-           :headers {"content-type" "text/html; charset=utf-8"}
-           :body result}
-          {:status 404
-           :body "not found"})))))

backend/src/app/http.clj

@@ -2,156 +2,165 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http
   (:require
    [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.config :as cfg]
-   [app.http.auth :as auth]
+   [app.http.doc :as doc]
    [app.http.errors :as errors]
    [app.http.middleware :as middleware]
    [app.metrics :as mtx]
-   [app.util.log4j :refer [update-thread-context!]]
    [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
    [integrant.core :as ig]
    [reitit.ring :as rr]
-   [ring.adapter.jetty9 :as jetty])
+   [yetti.adapter :as yt])
   (:import
    org.eclipse.jetty.server.Server
-   org.eclipse.jetty.server.handler.ErrorHandler
    org.eclipse.jetty.server.handler.StatisticsHandler))
 
+(declare wrap-router)
+
 (s/def ::handler fn?)
-(s/def ::ws (s/map-of ::us/string fn?))
-(s/def ::port ::cfg/http-server-port)
+(s/def ::router some?)
+(s/def ::port ::us/integer)
+(s/def ::host ::us/string)
 (s/def ::name ::us/string)
 
 (defmethod ig/pre-init-spec ::server [_]
-  (s/keys :req-un [::handler ::port]
-          :opt-un [::ws ::name ::mtx/metrics]))
+  (s/keys :req-un [::port]
+          :opt-un [::name ::mtx/metrics ::router ::handler ::host]))
 
 (defmethod ig/prep-key ::server
   [_ cfg]
-  (merge {:name "http"}
-         (d/without-nils cfg)))
+  (merge {:name "http"} (d/without-nils cfg)))
+
+(defn- instrument-metrics
+  [^Server server metrics]
+  (let [stats (doto (StatisticsHandler.)
+                (.setHandler (.getHandler server)))]
+    (.setHandler server stats)
+    (mtx/instrument-jetty! (:registry metrics) stats)
+    server))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [handler ws port name metrics] :as opts}]
-  (log/infof "Starting %s server on port %s." name port)
-  (let [pre-start (fn [^Server server]
-                    (let [handler (doto (ErrorHandler.)
-                                    (.setShowStacks true)
-                                    (.setServer server))]
-                      (.setErrorHandler server ^ErrorHandler handler)
-                      (when metrics
-                        (let [stats (new StatisticsHandler)]
-                          (.setHandler ^StatisticsHandler stats (.getHandler server))
-                          (.setHandler server stats)
-                          (mtx/instrument-jetty! (:registry metrics) stats)))))
-
-        options   (merge
-                   {:port port
-                    :h2c? true
-                    :join? false
-                    :allow-null-path-info true
-                    :configurator pre-start}
-                   (when (seq ws)
-                     {:websockets ws}))
-
-        server    (jetty/run-jetty handler options)]
-    (assoc opts :server server)))
+  [_ {:keys [handler router port name metrics host] :as opts}]
+  (l/info :msg "starting http server" :port port :host host :name name)
+  (let [options {:http/port port :http/host host}
+        handler (cond
+                  (fn? handler)  handler
+                  (some? router) (wrap-router router)
+                  :else (ex/raise :type :internal
+                                  :code :invalid-argument
+                                  :hint "Missing `handler` or `router` option."))
+        server  (-> (yt/server handler options)
+                    (cond-> metrics (instrument-metrics metrics)))]
+    (assoc opts :server (yt/start! server))))
 
 (defmethod ig/halt-key! ::server
   [_ {:keys [server name port] :as opts}]
-  (log/infof "Stoping %s server on port %s." name port)
-  (jetty/stop-server server))
+  (l/info :msg "stoping http server" :name name :port port)
+  (yt/stop! server))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Main Handler (Router)
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(declare create-router)
-
-(s/def ::rpc map?)
-(s/def ::session map?)
-(s/def ::metrics map?)
-(s/def ::google-auth map?)
-(s/def ::gitlab-auth map?)
-(s/def ::ldap-auth fn?)
-(s/def ::storage map?)
-(s/def ::assets map?)
-
-(defmethod ig/pre-init-spec ::router [_]
-  (s/keys :req-un [::rpc ::session ::metrics ::google-auth ::gitlab-auth ::storage ::assets]))
-
-(defmethod ig/init-key ::router
-  [_ cfg]
-  (let [handler (rr/ring-handler
-                 (create-router cfg)
-                 (rr/routes
-                  (rr/create-resource-handler {:path "/"})
-                  (rr/create-default-handler))
-                 {:middleware [middleware/server-timing]})]
+(defn- wrap-router
+  [router]
+  (let [default (rr/routes
+                 (rr/create-resource-handler {:path "/"})
+                 (rr/create-default-handler))
+        options {:middleware [middleware/server-timing]}
+        handler (rr/ring-handler router default options)]
     (fn [request]
       (try
         (handler request)
         (catch Throwable e
-          (try
-            (let [cdata (errors/get-error-context request e)]
-              (update-thread-context! cdata)
-              (log/errorf e "Unhandled exception: %s (id: %s)" (ex-message e) (str (:id cdata)))
-              {:status 500
-               :body "internal server error"})
-            (catch Throwable e
-              (log/errorf e "Unhandled exception: %s" (ex-message e))
-              {:status 500
-               :body "internal server error"})))))))
+          (l/error :hint "unexpected error processing request"
+                   ::l/context (errors/get-error-context request e)
+                   :query-string (:query-string request)
+                   :cause e)
+          {:status 500 :body "internal server error"})))))
 
-(defn- create-router
-  [{:keys [session rpc google-auth gitlab-auth github-auth metrics ldap-auth svgparse assets] :as cfg}]
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Http Router
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::rpc map?)
+(s/def ::session map?)
+(s/def ::oauth map?)
+(s/def ::storage map?)
+(s/def ::assets map?)
+(s/def ::feedback fn?)
+(s/def ::ws fn?)
+(s/def ::audit-http-handler fn?)
+(s/def ::debug map?)
+
+(defmethod ig/pre-init-spec ::router [_]
+  (s/keys :req-un [::rpc ::session ::mtx/metrics ::ws
+                   ::oauth ::storage ::assets ::feedback
+                   ::debug ::audit-http-handler]))
+
+(defmethod ig/init-key ::router
+  [_ {:keys [ws session rpc oauth metrics assets feedback debug] :as cfg}]
   (rr/router
    [["/metrics" {:get (:handler metrics)}]
-
     ["/assets" {:middleware [[middleware/format-response-body]
-                             [middleware/errors errors/handle]]}
+                             [middleware/errors errors/handle]
+                             [middleware/cookies]
+                             (:middleware session)]}
      ["/by-id/:id" {:get (:objects-handler assets)}]
      ["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
      ["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
 
-    ["/dbg"
-     ["/error-by-id/:id" {:get (:error-report-handler cfg)}]]
+    ["/dbg" {:middleware [[middleware/multipart-params]
+                          [middleware/params]
+                          [middleware/keyword-params]
+                          [middleware/format-response-body]
+                          [middleware/errors errors/handle]
+                          [middleware/cookies]
+                          [(:middleware session)]]}
+     ["" {:get (:index debug)}]
+     ["/error-by-id/:id" {:get (:retrieve-error debug)}]
+     ["/error/:id" {:get (:retrieve-error debug)}]
+     ["/error" {:get (:retrieve-error-list debug)}]
+     ["/file/data" {:get (:retrieve-file-data debug)
+                    :post (:upload-file-data debug)}]
+     ["/file/changes" {:get (:retrieve-file-changes debug)}]]
 
-    ["/api" {:middleware [[middleware/format-response-body]
+    ["/webhooks"
+     ["/sns" {:post (:sns-webhook cfg)}]]
+
+    ["/ws/notifications"
+     {:middleware [[middleware/params]
+                   [middleware/keyword-params]
+                   [middleware/format-response-body]
+                   [middleware/errors errors/handle]
+                   [middleware/cookies]
+                   [(:middleware session)]]
+      :get ws}]
+
+    ["/api" {:middleware [[middleware/cors]
                           [middleware/params]
                           [middleware/multipart-params]
                           [middleware/keyword-params]
+                          [middleware/format-response-body]
+                          [middleware/etag]
                           [middleware/parse-request-body]
                           [middleware/errors errors/handle]
                           [middleware/cookies]]}
 
-     ["/svg" {:post svgparse}]
+     ["/health" {:get (:health-check debug)}]
+     ["/_doc" {:get (doc/handler rpc)}]
+     ["/feedback" {:middleware [(:middleware session)]
+                   :post feedback}]
+     ["/auth/oauth/:provider" {:post (:handler oauth)}]
+     ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
 
-     ["/oauth"
-      ["/google" {:post (:auth-handler google-auth)}]
-      ["/google/callback" {:get (:callback-handler google-auth)}]
-
-      ["/gitlab" {:post (:auth-handler gitlab-auth)}]
-      ["/gitlab/callback" {:get (:callback-handler gitlab-auth)}]
-
-      ["/github" {:post (:auth-handler github-auth)}]
-      ["/github/callback" {:get (:callback-handler github-auth)}]]
-
-     ["/login" {:post #(auth/login-handler cfg %)}]
-     ["/logout" {:post #(auth/logout-handler cfg %)}]
-
-     ["/login-ldap" {:post ldap-auth}]
+     ["/audit/events" {:middleware [(:middleware session)]
+                       :post (:audit-http-handler cfg)}]
 
      ["/rpc" {:middleware [(:middleware session)]}
-      ["/query/:type" {:get (:query-handler rpc)}]
+      ["/query/:type" {:get (:query-handler rpc)
+                       :post (:query-handler rpc)}]
       ["/mutation/:type" {:post (:mutation-handler rpc)}]]]]))

backend/src/app/http/assets.clj

@@ -2,23 +2,20 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.assets
   "Assets related handlers."
   (:require
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
+   [app.common.uri :as u]
    [app.db :as db]
    [app.metrics :as mtx]
    [app.storage :as sto]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]
-   [lambdaisland.uri :as u]))
+   [integrant.core :as ig]))
 
 (def ^:private cache-max-age
   (dt/duration {:hours 24}))
@@ -52,7 +49,7 @@
       {:status 200
        :headers {"content-type" (:content-type mdata)
                  "cache-control" (str "max-age=" (inst-ms cache-max-age))}
-       :body (sto/get-object-data storage obj)}
+       :body (sto/get-object-bytes storage obj)}
 
       :s3
       (let [url (sto/get-object-url storage obj {:max-age signature-max-age})]

backend/src/app/http/auth.clj

@@ -1,31 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth
-  (:require
-   [app.http.session :as session]))
-
-(defn login-handler
-  [{:keys [session rpc] :as cfg} request]
-  (let [data    (:params request)
-        uagent  (get-in request [:headers "user-agent"])
-        method  (get-in rpc [:methods :mutation :login])
-        profile (method data)
-        id      (session/create! session {:profile-id (:id profile)
-                                          :user-agent uagent})]
-    {:status 200
-     :cookies (session/cookies session {:value id})
-     :body profile}))
-
-(defn logout-handler
-  [{:keys [session] :as cfg} request]
-  (session/delete! cfg request)
-  {:status 204
-   :cookies (session/cookies session {:value "" :max-age -1})
-   :body ""})

backend/src/app/http/auth/github.clj

@@ -1,171 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
-
-(ns app.http.auth.github
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.config :as cfg]
-   [app.http.session :as session]
-   [app.util.http :as http]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]
-   [lambdaisland.uri :as u]))
-
-(def base-github-uri
-  (u/uri "https://github.com"))
-
-(def base-api-github-uri
-  (u/uri "https://api.github.com"))
-
-(def authorize-uri
-  (assoc base-github-uri :path "/login/oauth/authorize"))
-
-(def token-url
-  (assoc base-github-uri :path "/login/oauth/access_token"))
-
-(def user-info-url
-  (assoc base-api-github-uri :path "/user"))
-
-(def scope "user:email")
-
-
-(defn- build-redirect-url
-  [cfg]
-  (let [public (u/uri (:public-uri cfg))]
-    (str (assoc public :path "/api/oauth/github/callback"))))
-
-(defn- get-access-token
-  [cfg state code]
-  (let [params {:client_id (:client-id cfg)
-                :client_secret (:client-secret cfg)
-                :code code
-                :state state
-                :redirect_uri (build-redirect-url cfg)}
-        req    {:method :post
-                :headers {"content-type" "application/x-www-form-urlencoded"
-                          "accept" "application/json"}
-                :uri  (str token-url)
-                :body (u/map->query-string params)}
-        res    (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-github
-                :context {:status (:status res)
-                          :body (:body res)}))
-    (try
-      (let [data (json/read-str (:body res))]
-        (get data "access_token"))
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from github access token request" e)
-        nil))))
-
-(defn- get-user-info
-  [token]
-  (let [req {:uri (str user-info-url)
-             :headers {"authorization" (str "token " token)}
-             :method :get}
-        res (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-github
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        {:email (get data "email")
-         :fullname (get data "name")})
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from github access token request" e)
-        nil))))
-
-(defn auth
-  [{:keys [tokens] :as cfg} _request]
-  (let [state  (tokens :generate
-                       {:iss :github-oauth
-                        :exp (dt/in-future "15m")})
-
-        params {:client_id (:client-id cfg/config)
-                :redirect_uri (build-redirect-url cfg)
-                :state state
-                :scope scope}
-        query (u/map->query-string params)
-        uri   (-> authorize-uri
-                  (assoc :query query))]
-    {:status 200
-     :body {:redirect-uri (str uri)}}))
-
-(defn callback
-  [{:keys [tokens rpc session] :as cfg} request]
-  (let [state (get-in request [:params :state])
-        _     (tokens :verify {:token state :iss :github-oauth})
-        info  (some->> (get-in request [:params :code])
-                       (get-access-token cfg state)
-                       (get-user-info))]
-
-    (when-not info
-      (ex/raise :type :authentication
-                :code :unable-to-authenticate-with-github))
-
-    (let [method-fn (get-in rpc [:methods :mutation :login-or-register])
-          profile   (method-fn {:email (:email info)
-                                :fullname (:fullname info)})
-          uagent    (get-in request [:headers "user-agent"])
-
-          token     (tokens :generate
-                            {:iss :auth
-                             :exp (dt/in-future "15m")
-                             :profile-id (:id profile)})
-
-          uri       (-> (u/uri (:public-uri cfg/config))
-                        (assoc :path "/#/auth/verify-token")
-                        (assoc :query (u/map->query-string {:token token})))
-
-          sid     (session/create! session {:profile-id (:id profile)
-                                            :user-agent uagent})]
-
-      {:status 302
-       :headers {"location" (str uri)}
-       :cookies (session/cookies session/cookies {:value sid})
-       :body ""})))
-
-;; --- ENTRY POINT
-
-(s/def ::client-id ::us/not-empty-string)
-(s/def ::client-secret ::us/not-empty-string)
-(s/def ::public-uri ::us/not-empty-string)
-(s/def ::session map?)
-(s/def ::tokens fn?)
-
-(defmethod ig/pre-init-spec :app.http.auth/github [_]
-  (s/keys :req-un [::public-uri
-                   ::session
-                   ::tokens]
-          :opt-un [::client-id
-                   ::client-secret]))
-
-(defn- default-handler
-  [_]
-  (ex/raise :type :not-found))
-
-(defmethod ig/init-key :app.http.auth/github
-  [_ cfg]
-  (if (and (:client-id cfg)
-           (:client-secret cfg))
-    {:auth-handler #(auth cfg %)
-     :callback-handler #(callback cfg %)}
-    {:auth-handler default-handler
-     :callback-handler default-handler}))
-

backend/src/app/http/auth/gitlab.clj

@@ -1,179 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.gitlab
-  (:require
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.http.session :as session]
-   [app.util.http :as http]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]
-   [lambdaisland.uri :as uri]))
-
-(def scope "read_user")
-
-(defn- build-redirect-url
-  [cfg]
-  (let [public (uri/uri (:public-uri cfg))]
-    (str (assoc public :path "/api/oauth/gitlab/callback"))))
-
-
-(defn- build-oauth-uri
-  [cfg]
-  (let [base-uri (uri/uri (:base-uri cfg))]
-    (assoc base-uri :path "/oauth/authorize")))
-
-
-(defn- build-token-url
-  [cfg]
-  (let [base-uri (uri/uri (:base-uri cfg))]
-    (str (assoc base-uri :path "/oauth/token"))))
-
-
-(defn- build-user-info-url
-  [cfg]
-  (let [base-uri (uri/uri (:base-uri cfg))]
-    (str (assoc base-uri :path "/api/v4/user"))))
-
-(defn- get-access-token
-  [cfg code]
-  (let [params {:client_id (:client-id cfg)
-                :client_secret (:client-secret cfg)
-                :code code
-                :grant_type "authorization_code"
-                :redirect_uri (build-redirect-url cfg)}
-        req    {:method :post
-                :headers {"content-type" "application/x-www-form-urlencoded"}
-                :uri (build-token-url cfg)
-                :body (uri/map->query-string params)}
-        res    (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-gitlab
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        (get data "access_token"))
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from gitlab access token request" e)
-        nil))))
-
-
-(defn- get-user-info
-  [cfg token]
-  (let [req {:uri (build-user-info-url cfg)
-             :headers {"Authorization" (str "Bearer " token)}
-             :method :get}
-        res (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-gitlab
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        ;; (clojure.pprint/pprint data)
-        {:email (get data "email")
-         :fullname (get data "name")})
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from gitlab access token request" e)
-        nil))))
-
-(defn auth
-  [{:keys [tokens] :as cfg} _request]
-  (let [token  (tokens :generate {:iss :gitlab-oauth
-                                  :exp (dt/in-future "15m")})
-
-        params {:client_id (:client-id cfg)
-                :redirect_uri (build-redirect-url cfg)
-                :response_type "code"
-                :state token
-                :scope scope}
-        query  (uri/map->query-string params)
-        uri    (-> (build-oauth-uri cfg)
-                   (assoc :query query))]
-    {:status 200
-     :body {:redirect-uri (str uri)}}))
-
-(defn callback
-  [{:keys [tokens rpc session] :as cfg} request]
-  (let [token (get-in request [:params :state])
-        _     (tokens :verify {:token token :iss :gitlab-oauth})
-        info  (some->> (get-in request [:params :code])
-                       (get-access-token cfg)
-                       (get-user-info cfg))]
-
-    (when-not info
-      (ex/raise :type :authentication
-                :code :unable-to-authenticate-with-gitlab))
-
-    (let [method-fn (get-in rpc [:methods :mutation :login-or-register])
-          profile   (method-fn {:email (:email info)
-                                :fullname (:fullname info)})
-          uagent    (get-in request [:headers "user-agent"])
-
-          token     (tokens :generate {:iss :auth
-                                       :exp (dt/in-future "15m")
-                                       :profile-id (:id profile)})
-
-          uri       (-> (uri/uri (:public-uri cfg))
-                        (assoc :path "/#/auth/verify-token")
-                        (assoc :query (uri/map->query-string {:token token})))
-
-          sid       (session/create! session {:profile-id (:id profile)
-                                              :user-agent uagent})]
-      {:status 302
-       :headers {"location" (str uri)}
-       :cookies (session/cookies session {:value sid})
-       :body ""})))
-
-
-(s/def ::client-id ::us/not-empty-string)
-(s/def ::client-secret ::us/not-empty-string)
-(s/def ::base-uri ::us/not-empty-string)
-(s/def ::public-uri ::us/not-empty-string)
-(s/def ::session map?)
-(s/def ::tokens fn?)
-
-(defmethod ig/pre-init-spec :app.http.auth/gitlab [_]
-  (s/keys :req-un [::public-uri
-                   ::session
-                   ::tokens]
-          :opt-un [::base-uri
-                   ::client-id
-                   ::client-secret]))
-
-
-(defmethod ig/prep-key :app.http.auth/gitlab
-  [_ cfg]
-  (d/merge {:base-uri "https://gitlab.com"}
-           (d/without-nils cfg)))
-
-(defn- default-handler
-  [_]
-  (ex/raise :type :not-found))
-
-(defmethod ig/init-key :app.http.auth/gitlab
-  [_ cfg]
-  (if (and (:client-id cfg)
-           (:client-secret cfg))
-    {:auth-handler #(auth cfg %)
-     :callback-handler #(callback cfg %)}
-    {:auth-handler default-handler
-     :callback-handler default-handler}))

backend/src/app/http/auth/google.clj

@@ -1,154 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.google
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.http.session :as session]
-   [app.util.http :as http]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]
-   [lambdaisland.uri :as uri]))
-
-(def base-goauth-uri "https://accounts.google.com/o/oauth2/v2/auth")
-
-(def scope
-  (str "email profile "
-       "https://www.googleapis.com/auth/userinfo.email "
-       "https://www.googleapis.com/auth/userinfo.profile "
-       "openid"))
-
-(defn- build-redirect-url
-  [cfg]
-  (let [public (uri/uri (:public-uri cfg))]
-    (str (assoc public :path "/api/oauth/google/callback"))))
-
-(defn- get-access-token
-  [cfg code]
-  (let [params {:code code
-                :client_id (:client-id cfg)
-                :client_secret (:client-secret cfg)
-                :redirect_uri (build-redirect-url cfg)
-                :grant_type "authorization_code"}
-        req    {:method :post
-                :headers {"content-type" "application/x-www-form-urlencoded"}
-                :uri "https://oauth2.googleapis.com/token"
-                :body (uri/map->query-string params)}
-        res    (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-google
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        (get data "access_token"))
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from google access token request" e)
-        nil))))
-
-(defn- get-user-info
-  [token]
-  (let [req {:uri "https://openidconnect.googleapis.com/v1/userinfo"
-             :headers {"Authorization" (str "Bearer " token)}
-             :method :get}
-        res (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-google
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        ;; (clojure.pprint/pprint data)
-        {:email (get data "email")
-         :fullname (get data "name")})
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from google access token request" e)
-        nil))))
-
-(defn- auth
-  [{:keys [tokens] :as cfg} _req]
-  (let [token  (tokens :generate {:iss :google-oauth :exp (dt/in-future "15m")})
-        params {:scope scope
-                :access_type "offline"
-                :include_granted_scopes true
-                :state token
-                :response_type "code"
-                :redirect_uri (build-redirect-url cfg)
-                :client_id (:client-id cfg)}
-        query  (uri/map->query-string params)
-        uri    (-> (uri/uri base-goauth-uri)
-                   (assoc :query query))]
-    {:status 200
-     :body {:redirect-uri (str uri)}}))
-
-(defn- callback
-  [{:keys [tokens rpc session] :as cfg} request]
-  (let [token (get-in request [:params :state])
-        _     (tokens :verify {:token token :iss :google-oauth})
-        info  (some->> (get-in request [:params :code])
-                       (get-access-token cfg)
-                       (get-user-info))]
-
-    (when-not info
-      (ex/raise :type :authentication
-                :code :unable-to-authenticate-with-google))
-
-    (let [method-fn (get-in rpc [:methods :mutation :login-or-register])
-          profile   (method-fn {:email (:email info)
-                                :fullname (:fullname info)})
-          uagent    (get-in request [:headers "user-agent"])
-          token     (tokens :generate {:iss :auth
-                                       :exp (dt/in-future "15m")
-                                       :profile-id (:id profile)})
-
-          uri       (-> (uri/uri (:public-uri cfg))
-                        (assoc :path "/#/auth/verify-token")
-                        (assoc :query (uri/map->query-string {:token token})))
-          sid       (session/create! session {:profile-id (:id profile)
-                                              :user-agent uagent})]
-      {:status 302
-       :headers {"location" (str uri)}
-       :cookies (session/cookies session {:value sid})
-       :body ""})))
-
-(s/def ::client-id ::us/not-empty-string)
-(s/def ::client-secret ::us/not-empty-string)
-(s/def ::public-uri ::us/not-empty-string)
-(s/def ::session map?)
-(s/def ::tokens fn?)
-
-(defmethod ig/pre-init-spec :app.http.auth/google [_]
-  (s/keys :req-un [::public-uri
-                   ::session
-                   ::tokens]
-          :opt-un [::client-id
-                   ::client-secret]))
-
-(defn- default-handler
-  [_]
-  (ex/raise :type :not-found))
-
-(defmethod ig/init-key :app.http.auth/google
-  [_ cfg]
-  (if (and (:client-id cfg)
-           (:client-secret cfg))
-    {:auth-handler #(auth cfg %)
-     :callback-handler #(callback cfg %)}
-    {:auth-handler default-handler
-     :callback-handler default-handler}))

backend/src/app/http/auth/ldap.clj

@@ -1,129 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.ldap
-  (:require
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
-   [app.http.session :as session]
-   [clj-ldap.client :as client]
-   [clojure.set :as set]
-   [clojure.spec.alpha :as s]
-   [clojure.string ]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]))
-
-(declare authenticate)
-(declare create-connection)
-(declare replace-several)
-
-
-(s/def ::host ::cfg/ldap-auth-host)
-(s/def ::port ::cfg/ldap-auth-port)
-(s/def ::ssl ::cfg/ldap-auth-ssl)
-(s/def ::starttls ::cfg/ldap-auth-starttls)
-(s/def ::user-query ::cfg/ldap-auth-user-query)
-(s/def ::base-dn ::cfg/ldap-auth-base-dn)
-(s/def ::username-attribute ::cfg/ldap-auth-username-attribute)
-(s/def ::email-attribute ::cfg/ldap-auth-email-attribute)
-(s/def ::fullname-attribute ::cfg/ldap-auth-fullname-attribute)
-(s/def ::avatar-attribute ::cfg/ldap-auth-avatar-attribute)
-
-(s/def ::rpc map?)
-(s/def ::session map?)
-
-(defmethod ig/pre-init-spec :app.http.auth/ldap
-  [_]
-  (s/keys
-   :req-un [::rpc ::session]
-   :opt-un [::host
-            ::port
-            ::ssl
-            ::starttls
-            ::username-attribute
-            ::base-dn
-            ::username-attribute
-            ::email-attribute
-            ::fullname-attribute
-            ::avatar-attribute]))
-
-(defmethod ig/init-key :app.http.auth/ldap
-  [_ {:keys [session rpc] :as cfg}]
-  (let [conn (create-connection cfg)]
-    (with-meta
-      (fn [request]
-        (let [data   (:body-params request)]
-          (when-some [info (authenticate (assoc cfg
-                                                :conn conn
-                                                :username (:email data)
-                                                :password (:password data)))]
-            (let [method-fn (get-in rpc [:methods :mutation :login-or-register])
-                  profile   (method-fn {:email (:email info)
-                                        :fullname (:fullname info)})
-                  uagent    (get-in request [:headers "user-agent"])
-                  sid       (session/create! session {:profile-id (:id profile)
-                                                      :user-agent uagent})]
-              {:status 200
-               :cookies (session/cookies session {:value sid})
-               :body profile}))))
-      {::conn conn})))
-
-(defmethod ig/halt-key! ::client
-  [_ handler]
-  (let [{:keys [::conn]} (meta handler)]
-    (when (realized? conn)
-      (.close @conn))))
-
-(defn- replace-several [s & {:as replacements}]
-  (reduce-kv clojure.string/replace s replacements))
-
-(defn- create-connection
-  [cfg]
-  (let [params (merge {:host {:address (:host cfg)
-                              :port (:port cfg)}}
-                      (-> cfg
-                          (select-keys [:ssl
-                                        :starttls
-                                        :ldap-bind-dn
-                                        :ldap-bind-password])
-                          (set/rename-keys {:ssl :ssl?
-                                            :starttls :startTLS?
-                                            :ldap-bind-dn  :bind-dn
-                                            :ldap-bind-password :password})))]
-    (delay
-      (try
-        (client/connect params)
-        (catch Exception e
-          (log/errorf e "Cannot connect to LDAP %s:%s"
-                      (:host cfg) (:port cfg)))))))
-
-
-(defn- authenticate
-  [{:keys [conn username password] :as cfg}]
-  (when-some [conn (some-> conn deref)]
-    (let [user-search-query (replace-several (:user-query cfg) "$username" username)
-          user-attributes   (-> cfg
-                                (select-keys [:username-attribute
-                                              :email-attribute
-                                              :fullname-attribute
-                                              :avatar-attribute])
-                                vals)]
-      (when-some [user-entry (-> conn
-                                 (client/search (:base-dn cfg)
-                                                {:filter user-search-query
-                                                 :sizelimit 1
-                                                 :attributes user-attributes})
-                                 (first))]
-        (when-not (client/bind? conn (:dn user-entry) password)
-          (ex/raise :type :authentication
-                    :code :wrong-credentials))
-        (set/rename-keys user-entry {(keyword (:avatar-attribute cfg)) :photo
-                                     (keyword (:fullname-attribute cfg)) :fullname
-                                     (keyword (:email-attribute cfg)) :email})))))
-

backend/src/app/http/awsns.clj

@@ -0,0 +1,196 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.awsns
+  "AWS SNS webhook handler for bounces."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.db :as db]
+   [app.db.sql :as sql]
+   [app.util.http :as http]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]
+   [jsonista.core :as j]))
+
+(declare parse-json)
+(declare parse-notification)
+(declare process-report)
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::handler
+  [_ cfg]
+  (fn [request]
+    (let [body  (parse-json (slurp (:body request)))
+          mtype (get body "Type")]
+      (cond
+        (= mtype "SubscriptionConfirmation")
+        (let [surl   (get body "SubscribeURL")
+              stopic (get body "TopicArn")]
+          (l/info :action "subscription received" :topic stopic :url surl)
+          (http/send! {:uri surl :method :post :timeout 10000}))
+
+        (= mtype "Notification")
+        (when-let [message (parse-json (get body "Message"))]
+          (let [notification (parse-notification cfg message)]
+            (process-report cfg notification)))
+
+        :else
+        (l/warn :hint "unexpected data received"
+                :report (pr-str body)))
+      {:status 200 :body ""})))
+
+(defn- parse-bounce
+  [data]
+  {:type        "bounce"
+   :kind        (str/lower (get data "bounceType"))
+   :category    (str/lower (get data "bounceSubType"))
+   :feedback-id (get data "feedbackId")
+   :timestamp   (get data "timestamp")
+   :recipients  (->> (get data "bouncedRecipients")
+                     (mapv (fn [item]
+                             {:email  (str/lower (get item "emailAddress"))
+                              :status (get item "status")
+                              :action (get item "action")
+                              :dcode  (get item "diagnosticCode")})))})
+
+(defn- parse-complaint
+  [data]
+  {:type          "complaint"
+   :user-agent    (get data "userAgent")
+   :kind          (get data "complaintFeedbackType")
+   :category      (get data "complaintSubType")
+   :timestamp     (get data "arrivalDate")
+   :feedback-id   (get data "feedbackId")
+   :recipients    (->> (get data "complainedRecipients")
+                       (mapv #(get % "emailAddress"))
+                       (mapv str/lower))})
+
+(defn- extract-headers
+  [mail]
+  (reduce (fn [acc item]
+            (let [key (get item "name")
+                  val (get item "value")]
+              (assoc acc (str/lower key) val)))
+          {}
+          (get mail "headers")))
+
+(defn- extract-identity
+  [{:keys [tokens] :as cfg} headers]
+  (let [tdata (get headers "x-penpot-data")]
+    (when-not (str/empty? tdata)
+      (let [result (tokens :verify {:token tdata :iss :profile-identity})]
+        (:profile-id result)))))
+
+(defn- parse-notification
+  [cfg message]
+  (let [type (get message "notificationType")
+        data (case type
+               "Bounce" (parse-bounce (get message "bounce"))
+               "Complaint" (parse-complaint (get message "complaint"))
+               {:type (keyword (str/lower type))
+                :message message})]
+    (when data
+      (let [mail (get message "mail")]
+        (when-not mail
+          (ex/raise :type :internal
+                    :code :incomplete-notification
+                    :hint "no email data received, please enable full headers report"))
+        (let [headers (extract-headers mail)
+              mail    {:destination (get mail "destination")
+                       :source      (get mail "source")
+                       :timestamp   (get mail "timestamp")
+                       :subject     (get-in mail ["commonHeaders" "subject"])
+                       :headers     headers}]
+          (assoc data
+                 :mail mail
+                 :profile-id (extract-identity cfg headers)))))))
+
+(defn- parse-json
+  [v]
+  (ex/ignoring
+   (j/read-value v)))
+
+(defn- register-bounce-for-profile
+  [{:keys [pool]} {:keys [type kind profile-id] :as report}]
+  (when (= kind "permanent")
+    (db/with-atomic [conn pool]
+      (db/insert! conn :profile-complaint-report
+                  {:profile-id profile-id
+                   :type (name type)
+                   :content (db/tjson report)})
+
+      ;; TODO: maybe also try to find profiles by mail and if exists
+      ;; register profile reports for them?
+      (doseq [recipient (:recipients report)]
+        (db/insert! conn :global-complaint-report
+                    {:email (:email recipient)
+                     :type (name type)
+                     :content (db/tjson report)}))
+
+      (let [profile (db/exec-one! conn (sql/select :profile {:id profile-id}))]
+        (when (some #(= (:email profile) (:email %)) (:recipients report))
+          ;; If the report matches the profile email, this means that
+          ;; the report is for itself, can be caused when a user
+          ;; registers with an invalid email or the user email is
+          ;; permanently rejecting receiving the email. In this case we
+          ;; have no option to mark the user as muted (and in this case
+          ;; the profile will be also inactive.
+          (db/update! conn :profile
+                      {:is-muted true}
+                      {:id profile-id}))))))
+
+(defn- register-complaint-for-profile
+  [{:keys [pool]} {:keys [type profile-id] :as report}]
+  (db/with-atomic [conn pool]
+    (db/insert! conn :profile-complaint-report
+                {:profile-id profile-id
+                 :type (name type)
+                 :content (db/tjson report)})
+
+    ;; TODO: maybe also try to find profiles by email and if exists
+    ;; register profile reports for them?
+    (doseq [email (:recipients report)]
+      (db/insert! conn :global-complaint-report
+                  {:email email
+                   :type (name type)
+                   :content (db/tjson report)}))
+
+    (let [profile (db/exec-one! conn (sql/select :profile {:id profile-id}))]
+      (when (some #(= % (:email profile)) (:recipients report))
+        ;; If the report matches the profile email, this means that
+        ;; the report is for itself, rare case but can happen; In this
+        ;; case just mark profile as muted (very rare case).
+        (db/update! conn :profile
+                    {:is-muted true}
+                    {:id profile-id})))))
+
+(defn- process-report
+  [cfg {:keys [type profile-id] :as report}]
+  (l/trace :action "processing report" :report (pr-str report))
+  (cond
+    ;; In this case we receive a bounce/complaint notification without
+    ;; confirmed identity, we just emit a warning but do nothing about
+    ;; it because this is not a normal case. All notifications should
+    ;; come with profile identity.
+    (nil? profile-id)
+    (l/warn :msg "a notification without identity received from AWS"
+            :report (pr-str report))
+
+    (= "bounce" type)
+    (register-bounce-for-profile cfg report)
+
+    (= "complaint" type)
+    (register-complaint-for-profile cfg report)
+
+    :else
+    (l/warn :msg "unrecognized report received from AWS"
+            :report (pr-str report))))
+
+

backend/src/app/http/debug.clj

@@ -0,0 +1,212 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.debug
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.rpc.mutations.files :as m.files]
+   [app.rpc.queries.profile :as profile]
+   [app.util.blob :as blob]
+   [app.util.template :as tmpl]
+   [app.util.time :as dt]
+   [clojure.java.io :as io]
+   [cuerdas.core :as str]
+   [datoteka.core :as fs]
+   [fipp.edn :as fpp]
+   [integrant.core :as ig]))
+
+;; (selmer.parser/cache-off!)
+
+(defn authorized?
+  [pool {:keys [profile-id]}]
+  (or (= "devenv" (cf/get :host))
+      (let [profile (ex/ignoring (profile/retrieve-profile-data pool profile-id))
+            admins  (or (cf/get :admins) #{})]
+        (contains? admins (:email profile)))))
+
+(defn index
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  {:status 200
+   :headers {"content-type" "text/html"}
+   :body (-> (io/resource "templates/debug.tmpl")
+             (tmpl/render {}))})
+
+
+(def sql:retrieve-range-of-changes
+  "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
+
+(def sql:retrieve-single-change
+  "select revn, changes, data from file_change where file_id=? and revn = ?")
+
+(defn prepare-response
+  [{:keys [params] :as request} body]
+  (when-not body
+    (ex/raise :type :not-found
+              :code :enpty-data
+              :hint "empty response"))
+
+  (cond-> {:status 200
+           :headers {"content-type" "application/transit+json"}
+           :body body}
+    (contains? params :download)
+    (update :headers assoc "content-disposition" "attachment")))
+
+(defn retrieve-file-data
+  [{:keys [pool]} {:keys [params] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (let [file-id (some-> (get-in request [:params :file-id]) uuid/uuid)
+        revn    (some-> (get-in request [:params :revn]) d/parse-integer)]
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments))
+
+    (let [data (if (integer? revn)
+                 (some-> (db/exec-one! pool [sql:retrieve-single-change file-id revn]) :data)
+                 (some-> (db/get-by-id pool :file file-id) :data))]
+      (if (contains? params :download)
+        (-> (prepare-response request data)
+            (update :headers assoc "content-type" "application/octet-stream"))
+        (prepare-response request (some-> data blob/decode))))))
+
+(defn upload-file-data
+  [{:keys [pool]} {:keys [profile-id params] :as request}]
+  (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
+        data       (some-> params :file :tempfile fs/slurp-bytes blob/decode)]
+
+    (if (and data project-id)
+      (let [fname (str "imported-file-" (dt/now))]
+        (m.files/create-file pool {:id (uuid/next)
+                                   :name fname
+                                   :project-id project-id
+                                   :profile-id profile-id
+                                   :data data})
+        {:status 200
+         :body "OK"})
+      {:status 500
+       :body "error"})))
+
+(defn retrieve-file-changes
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (let [file-id (some-> (get-in request [:params :id]) uuid/uuid)
+        revn    (or (get-in request [:params :revn]) "latest")]
+
+    (when (or (not file-id) (not revn))
+      (ex/raise :type :validation
+                :code :invalid-arguments
+                :hint "missing arguments"))
+
+    (cond
+      (d/num-string? revn)
+      (let [item (db/exec-one! pool [sql:retrieve-single-change file-id (d/parse-integer revn)])]
+        (prepare-response request (some-> item :changes blob/decode vec)))
+
+      (str/includes? revn ":")
+      (let [[start end] (->> (str/split revn #":")
+                             (map str/trim)
+                             (map d/parse-integer))
+            items       (db/exec! pool [sql:retrieve-range-of-changes file-id start end])]
+        (prepare-response request
+                          (some->> items
+                                   (map :changes)
+                                   (map blob/decode)
+                                   (mapcat identity)
+                                   (vec))))
+      :else
+      (ex/raise :type :validation :code :invalid-arguments))))
+
+
+(defn retrieve-error
+  [{:keys [pool]} request]
+  (letfn [(parse-id [request]
+            (let [id (get-in request [:path-params :id])
+                  id (us/uuid-conformer id)]
+              (when (uuid? id)
+                id)))
+
+          (retrieve-report [id]
+            (ex/ignoring
+             (some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject)))
+
+          (render-template [report]
+            (let [context (dissoc report
+                                  :trace :cause :params :data :spec-problems
+                                  :spec-explain :spec-value :error :explain :hint)
+                  params  {:context (with-out-str (fpp/pprint context {:width 300}))
+                           :hint    (:hint report)
+                           :spec-explain  (:spec-explain report)
+                           :spec-problems (:spec-problems report)
+                           :spec-value    (:spec-value report)
+                           :data          (:data report)
+                           :trace         (or (:trace report)
+                                              (some-> report :error :trace))
+                           :params        (:params report)}]
+              (-> (io/resource "templates/error-report.tmpl")
+                  (tmpl/render params))))
+          ]
+
+    (when-not (authorized? pool request)
+      (ex/raise :type :authentication
+                :code :only-admins-allowed))
+
+    (let [result (some-> (parse-id request)
+                         (retrieve-report)
+                         (render-template))]
+      (if result
+        {:status 200
+         :headers {"content-type" "text/html; charset=utf-8"
+                   "x-robots-tag" "noindex"}
+         :body result}
+        {:status 404
+         :body "not found"}))))
+
+(def sql:error-reports
+  "select id, created_at from server_error_report order by created_at desc limit 100")
+
+(defn retrieve-error-list
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+  (let [items (db/exec! pool [sql:error-reports])
+        items (map #(update % :created-at dt/format-instant :rfc1123) items)]
+    {:status 200
+     :headers {"content-type" "text/html; charset=utf-8"
+               "x-robots-tag" "noindex"}
+     :body (-> (io/resource "templates/error-list.tmpl")
+               (tmpl/render {:items items}))}))
+
+(defn health-check
+  "Mainly a task that performs a health check."
+  [{:keys [pool]} _]
+  (db/with-atomic [conn pool]
+    (db/exec-one! conn ["select count(*) as count from server_prop;"])
+    {:status 200 :body "Ok"}))
+
+(defmethod ig/init-key ::handlers
+  [_ cfg]
+  {:index (partial index cfg)
+   :health-check (partial health-check cfg)
+   :retrieve-file-data (partial retrieve-file-data cfg)
+   :retrieve-file-changes (partial retrieve-file-changes cfg)
+   :retrieve-error (partial retrieve-error cfg)
+   :retrieve-error-list (partial retrieve-error-list cfg)
+   :upload-file-data (partial upload-file-data cfg)})

backend/src/app/http/doc.clj

@@ -0,0 +1,53 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.doc
+  "API autogenerated documentation."
+  (:require
+   [app.common.data :as d]
+   [app.config :as cf]
+   [app.util.services :as sv]
+   [app.util.template :as tmpl]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [pretty-spec.core :as ps]))
+
+(defn get-spec-str
+  [k]
+  (with-out-str
+    (ps/pprint (s/form k)
+               {:ns-aliases {"clojure.spec.alpha" "s"
+                             "clojure.core.specs.alpha" "score"
+                             "clojure.core" nil}})))
+
+(defn prepare-context
+  [rpc]
+  (letfn [(gen-doc [type [name f]]
+            (let [mdata (meta f)]
+              ;; (prn name mdata)
+              {:type (d/name type)
+               :name (d/name name)
+               :auth (:auth mdata true)
+               :docs (::sv/docs mdata)
+               :spec (get-spec-str (::sv/spec mdata))}))]
+    {:query-methods
+     (into []
+           (map (partial gen-doc :query))
+           (->> rpc :methods :query (sort-by first)))
+     :mutation-methods
+     (into []
+           (map (partial gen-doc :mutation))
+           (->> rpc :methods :mutation (sort-by first)))}))
+
+(defn handler
+  [rpc]
+  (let [context (prepare-context rpc)]
+    (if (contains? cf/flags :backend-api-doc)
+      (fn [_]
+        {:status 200
+         :body (-> (io/resource "api-doc.tmpl")
+                   (tmpl/render context))})
+      (constantly {:status 404 :body ""}))))

backend/src/app/http/errors.clj

@@ -2,46 +2,44 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.errors
   "A errors handling for the http server."
   (:require
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.util.log4j :refer [update-thread-context!]]
-   [clojure.tools.logging :as log]
-   [cuerdas.core :as str]
-   [expound.alpha :as expound]))
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]))
 
-(defn- explain-error
-  [error]
-  (with-out-str
-    (expound/printer (:data error))))
+(defn- parse-client-ip
+  [{:keys [headers] :as request}]
+  (or (some-> (get headers "x-forwarded-for") (str/split ",") first)
+      (get headers "x-real-ip")
+      (get request :remote-addr)))
 
 (defn get-error-context
   [request error]
-  (let [edata (ex-data error)]
+  (let [data (ex-data error)]
     (merge
-     {:id      (uuid/next)
-      :path    (:uri request)
-      :method  (:request-method request)
-      :params  (:params request)
-      :version (:full cfg/version)
-      :host    (:public-uri cfg/config)
-      :class   (.getCanonicalName ^java.lang.Class (class error))
-      :hint    (ex-message error)
-      :data    edata}
+     {:path          (:uri request)
+      :method        (:request-method request)
+      :hint          (ex-message error)
+      :params        (:params request)
+
+      :spec-problems (some->> data ::s/problems (take 10) seq vec)
+      :spec-value    (some->> data ::s/value)
+      :data          (some-> data (dissoc ::s/problems ::s/value ::s/spec))
+      :ip-addr       (parse-client-ip request)
+      :profile-id    (:profile-id request)}
 
      (let [headers (:headers request)]
        {:user-agent (get headers "user-agent")
         :frontend-version (get headers "x-frontend-version" "unknown")})
 
-     (when (and (map? edata) (:data edata))
-       {:explain (explain-error edata)}))))
+     (when (and data (::s/problems data))
+       {:spec-explain (us/pretty-explain data)}))))
 
 (defmulti handle-exception
   (fn [err & _rest]
@@ -53,34 +51,31 @@
   [err _]
   {:status 401 :body (ex-data err)})
 
+(defmethod handle-exception :restriction
+  [err _]
+  {:status 400 :body (ex-data err)})
+
 (defmethod handle-exception :validation
-  [err req]
-  (let [header (get-in req [:headers "accept"])
-        edata  (ex-data err)]
-    (if (and (= :spec-validation (:code edata))
-             (str/starts-with? header "text/html"))
-      {:status 400
-       :headers {"content-type" "text/html"}
-       :body (str "<pre style='font-size:16px'>"
-                  (explain-error edata)
-                  "</pre>\n")}
-      {:status 400
-       :body   (cond-> edata
-                 (map? (:data edata))
-                 (-> (assoc :explain (explain-error edata))
-                     (dissoc :data)))})))
+  [err _]
+  (let [data    (ex-data err)
+        explain (us/pretty-explain data)]
+    {:status 400
+     :body (-> data
+               (dissoc ::s/problems)
+               (dissoc ::s/value)
+               (cond-> explain (assoc :explain explain)))}))
 
 (defmethod handle-exception :assertion
   [error request]
-  (let [edata (ex-data error)
-        cdata (get-error-context request error)]
-    (update-thread-context! cdata)
-    (log/errorf error "Internal error: assertion (id: %s)" (str (:id cdata)))
+  (let [edata (ex-data error)]
+    (l/error ::l/raw (ex-message error)
+             ::l/context (get-error-context request error)
+             :cause error)
+
     {:status 500
      :body {:type :server-error
-            :data (-> edata
-                      (assoc :explain (explain-error edata))
-                      (dissoc :data))}}))
+            :code :assertion
+            :data (dissoc edata ::s/problems ::s/value ::s/spec)}}))
 
 (defmethod handle-exception :not-found
   [err _]
@@ -88,15 +83,50 @@
 
 (defmethod handle-exception :default
   [error request]
-  (let [cdata (get-error-context request error)]
-    (update-thread-context! cdata)
-    (log/errorf error "Internal error: %s (id: %s)"
-                (ex-message error)
-                (str (:id cdata)))
-    {:status 500
-     :body {:type :server-error
-            :hint (ex-message error)
-            :data (ex-data error)}}))
+  (let [edata (ex-data error)]
+    ;; NOTE: this is a special case for the idle-in-transaction error;
+    ;; when it happens, the connection is automatically closed and
+    ;; next-jdbc combines the two errors in a single ex-info. We only
+    ;; need the :handling error, because the :rollback error will be
+    ;; always "connection closed".
+    (if (and (ex/exception? (:rollback edata))
+             (ex/exception? (:handling edata)))
+      (handle-exception (:handling edata) request)
+      (do
+        (l/error ::l/raw (ex-message error)
+                 ::l/context (get-error-context request error)
+                 :cause error)
+        {:status 500
+         :body {:type :server-error
+                :code :unexpected
+                :hint (ex-message error)
+                :data edata}}))))
+
+(defmethod handle-exception org.postgresql.util.PSQLException
+  [error request]
+  (let [state (.getSQLState ^java.sql.SQLException error)]
+    (l/error ::l/raw (ex-message error)
+             ::l/context (get-error-context request error)
+             :cause error)
+    (cond
+      (= state "57014")
+      {:status 504
+       :body {:type :server-timeout
+              :code :statement-timeout
+              :hint (ex-message error)}}
+
+      (= state "25P03")
+      {:status 504
+       :body {:type :server-timeout
+              :code :idle-in-transaction-timeout
+              :hint (ex-message error)}}
+
+      :else
+      {:status 500
+       :body {:type :server-error
+              :code :psql-exception
+              :hint (ex-message error)
+              :state state}})))
 
 (defn handle
   [error req]

backend/src/app/http/feedback.clj

@@ -0,0 +1,71 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.feedback
+  "A general purpose feedback module."
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.emails :as eml]
+   [app.rpc.queries.profile :as profile]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(declare send-feedback)
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::handler
+  [_ {:keys [pool] :as scfg}]
+  (let [ftoken   (cf/get :feedback-token ::no-token)
+        enabled  (contains? cf/flags :user-feedback)]
+    (fn [{:keys [profile-id] :as request}]
+      (let [token  (get-in request [:headers "x-feedback-token"])
+            params (d/merge (:params request)
+                            (:body-params request))]
+
+        (when-not enabled
+          (ex/raise :type :validation
+                    :code :feedback-disabled
+                    :hint "feedback module is disabled"))
+
+        (cond
+          (uuid? profile-id)
+          (let [profile (profile/retrieve-profile-data pool profile-id)
+                params  (assoc params :from (:email profile))]
+            (when-not (:is-muted profile)
+              (send-feedback pool profile params)))
+
+          (= token ftoken)
+          (send-feedback scfg nil params))
+
+        {:status 204 :body ""}))))
+
+(s/def ::content ::us/string)
+(s/def ::from    ::us/email)
+(s/def ::subject ::us/string)
+
+(s/def ::feedback
+  (s/keys :req-un [::from ::subject ::content]))
+
+(defn send-feedback
+  [pool profile params]
+  (let [params      (us/conform ::feedback params)
+        destination (cf/get :feedback-destination)]
+    (eml/send! {::eml/conn pool
+                ::eml/factory eml/feedback
+                :from     destination
+                :to       destination
+                :profile  profile
+                :reply-to (:from params)
+                :email    (:from params)
+                :subject  (:subject params)
+                :content  (:content params)})
+    nil))

backend/src/app/http/middleware.clj

@@ -2,21 +2,22 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.middleware
   (:require
-   [app.metrics :as mtx]
+   [app.common.logging :as l]
+   [app.common.transit :as t]
+   [app.config :as cf]
    [app.util.json :as json]
-   [app.util.transit :as t]
-   [clojure.java.io :as io]
+   [buddy.core.codecs :as bc]
+   [buddy.core.hash :as bh]
+   [ring.core.protocols :as rp]
    [ring.middleware.cookies :refer [wrap-cookies]]
    [ring.middleware.keyword-params :refer [wrap-keyword-params]]
    [ring.middleware.multipart-params :refer [wrap-multipart-params]]
-   [ring.middleware.params :refer [wrap-params]]))
+   [ring.middleware.params :refer [wrap-params]]
+   [yetti.adapter :as yt]))
 
 (defn wrap-server-timing
   [handler]
@@ -35,55 +36,80 @@
               (t/read! reader)))
 
           (parse-json [body]
-            (let [reader (io/reader body)]
-              (json/read reader)))
-
-          (parse [type body]
-            (try
-              (case type
-                :json (parse-json body)
-                :transit (parse-transit body))
-              (catch Exception e
-                (let [data {:type :parse
-                            :hint "unable to parse request body"
-                            :message (ex-message e)}]
-                  {:status 400
-                   :headers {"content-type" "application/transit+json"}
-                   :body (t/encode-str data {:type :json-verbose})}))))]
-
+            (json/read body))]
     (fn [{:keys [headers body] :as request}]
-      (let [ctype (get headers "content-type")]
-        (handler
-         (case ctype
-           "application/transit+json"
-           (let [params (parse :transit body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+      (try
+        (let [ctype (get headers "content-type")]
+          (handler (case ctype
+                     "application/transit+json"
+                     (let [params (parse-transit body)]
+                       (-> request
+                           (assoc :body-params params)
+                           (update :params merge params)))
 
-           "application/json"
-           (let [params (parse :json body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+                     "application/json"
+                     (let [params (parse-json body)]
+                       (-> request
+                           (assoc :body-params params)
+                           (update :params merge params)))
 
-           request))))))
+                     request)))
+        (catch Exception e
+          (let [data {:type :validation
+                      :code :unable-to-parse-request-body
+                      :hint "malformed params"}]
+            (l/error :hint (ex-message e) :cause e)
+            {:status 400
+             :headers {"content-type" "application/transit+json"}
+             :body (t/encode-str data {:type :json-verbose})}))))))
 
 (def parse-request-body
   {:name ::parse-request-body
    :compile (constantly wrap-parse-request-body)})
 
+(defn buffered-output-stream
+  "Returns a buffered output stream that ignores flush calls. This is
+  needed because transit-java calls flush very aggresivelly on each
+  object write."
+  [^java.io.OutputStream os ^long chunk-size]
+  (proxy [java.io.BufferedOutputStream] [os (int chunk-size)]
+    ;; Explicitly do not forward flush
+    (flush [])
+    (close []
+      (proxy-super flush)
+      (proxy-super close))))
+
+(def ^:const buffer-size (:http/output-buffer-size yt/base-defaults))
+
+(defn- transit-streamable-body
+  [data opts]
+  (reify rp/StreamableResponseBody
+    (write-body-to-stream [_ _ output-stream]
+      ;; Use the same buffer as jetty output buffer size
+      (try
+        (with-open [bos (buffered-output-stream output-stream buffer-size)]
+          (let [tw (t/writer bos opts)]
+            (t/write! tw data)))
+        (catch org.eclipse.jetty.io.EofException _cause
+          ;; Do nothing, EOF means client closes connection abruptly
+          nil)
+        (catch Throwable cause
+          (l/warn :hint "unexpected error on encoding response"
+                  :cause cause))))))
+
 (defn- impl-format-response-body
-  [response]
-  (let [body (:body response)
-        type :json-verbose]
+  [response {:keys [query-params] :as request}]
+  (let [body   (:body response)
+        opts   {:type (if (contains? query-params "transit_verbose") :json-verbose :json)}]
+
     (cond
+      (:ws response)
+      response
+
       (coll? body)
       (-> response
-          (assoc :body (t/encode body {:type type}))
-          (update :headers assoc
-                  "content-type"
-                  "application/transit+json"))
+          (update :headers assoc "content-type" "application/transit+json")
+          (assoc :body (transit-streamable-body body opts)))
 
       (nil? body)
       (assoc response :status 204 :body "")
@@ -96,7 +122,7 @@
   (fn [request]
     (let [response (handler request)]
       (cond-> response
-        (map? response) (impl-format-response-body)))))
+        (map? response) (impl-format-response-body request)))))
 
 (def format-response-body
   {:name ::format-response-body
@@ -114,13 +140,6 @@
   {:name ::errors
    :compile (constantly wrap-errors)})
 
-(def metrics
-  {:name ::metrics
-   :wrap (fn [handler]
-           (mtx/wrap-counter handler {:id "http__requests_counter"
-                                      :help "Absolute http requests counter."}))})
-
-
 (def cookies
   {:name ::cookies
    :compile (constantly wrap-cookies)})
@@ -140,3 +159,63 @@
 (def server-timing
   {:name ::server-timing
    :compile (constantly wrap-server-timing)})
+
+(defn wrap-etag
+  [handler]
+  (letfn [(encode [data]
+            (when (string? data)
+              (str "W/\"" (-> data bh/blake2b-128 bc/bytes->hex) "\"")))]
+    (fn [{method :request-method headers :headers :as request}]
+      (cond-> (handler request)
+        (= :get method)
+        (as-> $ (if-let [etag (-> $ :body meta :etag encode)]
+                  (cond-> (update $ :headers assoc "etag" etag)
+                    (= etag (get headers "if-none-match"))
+                    (-> (assoc :body "")
+                        (assoc :status 304)))
+                  $))))))
+
+(def etag
+  {:name ::etag
+   :compile (constantly wrap-etag)})
+
+(defn activity-logger
+  [handler]
+  (let [logger "penpot.profile-activity"]
+    (fn [{:keys [headers] :as request}]
+      (let [ip-addr    (get headers "x-forwarded-for")
+            profile-id (:profile-id request)
+            qstring    (:query-string request)]
+        (l/info ::l/async true
+                ::l/logger logger
+                :ip-addr ip-addr
+                :profile-id profile-id
+                :uri (str (:uri request) (when qstring (str "?" qstring)))
+                :method (name (:request-method request)))
+        (handler request)))))
+
+(defn- wrap-cors
+  [handler]
+  (if-not (contains? cf/flags :cors)
+    handler
+    (letfn [(add-cors-headers [response request]
+              (-> response
+                  (update
+                   :headers
+                   (fn [headers]
+                     (-> headers
+                         (assoc "access-control-allow-origin" (get-in request [:headers "origin"]))
+                         (assoc "access-control-allow-methods" "GET,POST,DELETE,OPTIONS,PUT,HEAD,PATCH")
+                         (assoc "access-control-allow-credentials" "true")
+                         (assoc "access-control-expose-headers" "x-requested-with, content-type, cookie")
+                         (assoc "access-control-allow-headers" "x-frontend-version, content-type, accept, x-requested-width"))))))]
+      (fn [request]
+        (if (= (:request-method request) :options)
+          (-> {:status 200 :body ""}
+              (add-cors-headers request))
+          (let [response (handler request)]
+            (add-cors-headers response request)))))))
+
+(def cors
+  {:name ::cors
+   :compile (constantly wrap-cors)})

backend/src/app/http/oauth.clj

@@ -0,0 +1,397 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.oauth
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.common.uri :as u]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.loggers.audit :as audit]
+   [app.rpc.queries.profile :as profile]
+   [app.util.http :as http]
+   [app.util.time :as dt]
+   [clojure.data.json :as json]
+   [clojure.set :as set]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+(defn- build-redirect-uri
+  [{:keys [provider] :as cfg}]
+  (let [public (u/uri (:public-uri cfg))]
+    (str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
+
+(defn- build-auth-uri
+  [{:keys [provider] :as cfg} state]
+  (let [params {:client_id (:client-id provider)
+                :redirect_uri (build-redirect-uri cfg)
+                :response_type "code"
+                :state state
+                :scope (str/join " " (:scopes provider []))}
+        query  (u/map->query-string params)]
+    (-> (u/uri (:auth-uri provider))
+        (assoc :query query)
+        (str))))
+
+(defn retrieve-access-token
+  [{:keys [provider] :as cfg} code]
+  (try
+    (let [params {:client_id (:client-id provider)
+                  :client_secret (:client-secret provider)
+                  :code code
+                  :grant_type "authorization_code"
+                  :redirect_uri (build-redirect-uri cfg)}
+          req    {:method :post
+                  :headers {"content-type" "application/x-www-form-urlencoded"}
+                  :uri (:token-uri provider)
+                  :body (u/map->query-string params)}
+          res    (http/send! req)]
+      (when (= 200 (:status res))
+        (let [data (json/read-str (:body res))]
+          {:token (get data "access_token")
+           :type  (get data "token_type")})))
+    (catch Exception e
+      (l/warn :hint "unexpected error on retrieve-access-token" :cause e)
+      nil)))
+
+(defn- qualify-props
+  [provider props]
+  (reduce-kv (fn [result k v]
+               (assoc result (keyword (:name provider) (name k)) v))
+             {}
+             props))
+
+(defn- retrieve-user-info
+  [{:keys [provider] :as cfg} tdata]
+  (try
+    (let [req {:uri (:user-uri provider)
+               :headers {"Authorization" (str (:type tdata) " " (:token tdata))}
+               :timeout 6000
+               :method :get}
+          res (http/send! req)]
+
+      (when (= 200 (:status res))
+        (let [info (json/read-str (:body res) :key-fn keyword)]
+          {:backend (:name provider)
+           :email (:email info)
+           :fullname (:name info)
+           :props (->> (dissoc info :name :email)
+                       (qualify-props provider))})))
+    (catch Exception e
+      (l/warn :hint "unexpected exception on retrieve-user-info" :cause e)
+      nil)))
+
+(s/def ::backend ::us/not-empty-string)
+(s/def ::email ::us/not-empty-string)
+(s/def ::fullname ::us/not-empty-string)
+(s/def ::props (s/map-of ::us/keyword any?))
+
+(s/def ::info
+  (s/keys :req-un [::backend
+                   ::email
+                   ::fullname
+                   ::props]))
+
+(defn retrieve-info
+  [{:keys [tokens provider] :as cfg} request]
+  (let [state  (get-in request [:params :state])
+        state  (tokens :verify {:token state :iss :oauth})
+        info   (some->> (get-in request [:params :code])
+                        (retrieve-access-token cfg)
+                        (retrieve-user-info cfg))]
+
+    (when-not (s/valid? ::info info)
+      (l/warn :hint "received incomplete profile info object (please set correct scopes)"
+              :info (pr-str info))
+      (ex/raise :type :internal
+                :code :unable-to-auth
+                :hint "no user info"))
+
+    ;; If the provider is OIDC, we can proceed to check
+    ;; roles if they are defined.
+    (when (and (= "oidc" (:name provider))
+               (seq (:roles provider)))
+      (let [provider-roles (into #{} (:roles provider))
+            profile-roles  (let [attr  (cf/get :oidc-roles-attr :roles)
+                                 roles (get info attr)]
+                             (cond
+                               (string? roles) (into #{} (str/words roles))
+                               (vector? roles) (into #{} roles)
+                               :else #{}))]
+
+        ;; check if profile has a configured set of roles
+        (when-not (set/subset? provider-roles profile-roles)
+          (ex/raise :type :internal
+                    :code :unable-to-auth
+                    :hint "not enough permissions"))))
+
+    (cond-> info
+      (some? (:invitation-token state))
+      (assoc :invitation-token (:invitation-token state))
+
+      ;; If state token comes with props, merge them. The state token
+      ;; props can contain pm_ and utm_ prefixed query params.
+      (map? (:props state))
+      (update :props merge (:props state)))))
+
+;; --- HTTP HANDLERS
+
+(defn extract-utm-props
+  "Extracts additional data from user params."
+  [params]
+  (reduce-kv (fn [params k v]
+               (let [sk (name k)]
+                 (cond-> params
+                   (str/starts-with? sk "utm_")
+                   (assoc (->> sk str/kebab (keyword "penpot")) v))))
+             {}
+             params))
+
+(defn- retrieve-profile
+  [{:keys [pool] :as cfg} info]
+  (with-open [conn (db/open pool)]
+    (some->> (:email info)
+             (profile/retrieve-profile-data-by-email conn)
+             (profile/populate-additional-data conn)
+             (profile/decode-profile-row))))
+
+(defn- redirect-response
+  [uri]
+  {:status 302
+   :headers {"location" (str uri)}
+   :body ""})
+
+(defn- generate-error-redirect
+  [cfg error]
+  (let [uri (-> (u/uri (:public-uri cfg))
+                (assoc :path "/#/auth/login")
+                (assoc :query (u/map->query-string {:error "unable-to-auth" :hint (ex-message error)})))]
+    (redirect-response uri)))
+
+(defn- generate-redirect
+  [{:keys [tokens session audit] :as cfg} request info profile]
+  (if profile
+    (let [sxf    ((:create session) (:id profile))
+          token  (or (:invitation-token info)
+                     (tokens :generate {:iss :auth
+                                        :exp (dt/in-future "15m")
+                                        :profile-id (:id profile)}))
+          params {:token token}
+
+          uri    (-> (u/uri (:public-uri cfg))
+                     (assoc :path "/#/auth/verify-token")
+                     (assoc :query (u/map->query-string params)))]
+
+      (when (fn? audit)
+        (audit :cmd :submit
+               :type "mutation"
+               :name "login"
+               :profile-id (:id profile)
+               :ip-addr (audit/parse-client-ip request)
+               :props (audit/profile->props profile)))
+
+      (->> (redirect-response uri)
+           (sxf request)))
+    (let [info   (assoc info
+                        :iss :prepared-register
+                        :is-active true
+                        :exp (dt/in-future {:hours 48}))
+          token  (tokens :generate info)
+          params (d/without-nils
+                  {:token token
+                   :fullname (:fullname info)})
+          uri    (-> (u/uri (:public-uri cfg))
+                     (assoc :path "/#/auth/register/validate")
+                     (assoc :query (u/map->query-string params)))]
+      (redirect-response uri))))
+
+(defn- auth-handler
+  [{:keys [tokens] :as cfg} {:keys [params] :as request}]
+  (let [invitation (:invitation-token params)
+        props      (extract-utm-props params)
+        state      (tokens :generate
+                           {:iss :oauth
+                            :invitation-token invitation
+                            :props props
+                            :exp (dt/in-future "15m")})
+        uri        (build-auth-uri cfg state)]
+    {:status 200
+     :body {:redirect-uri uri}}))
+
+(defn- callback-handler
+  [cfg request]
+  (try
+    (let [info     (retrieve-info cfg request)
+          profile  (retrieve-profile cfg info)]
+      (generate-redirect cfg request info profile))
+    (catch Exception e
+      (l/warn :hint "error on oauth process"
+              :cause e)
+      (generate-error-redirect cfg e))))
+
+;; --- INIT
+
+(declare initialize)
+
+(s/def ::public-uri ::us/not-empty-string)
+(s/def ::session map?)
+(s/def ::tokens fn?)
+(s/def ::rpc map?)
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::public-uri ::session ::tokens ::rpc ::db/pool]))
+
+(defn wrap-handler
+  [cfg handler]
+  (fn [request]
+    (let [provider (get-in request [:path-params :provider])
+          provider (get-in @cfg [:providers provider])]
+      (when-not provider
+        (ex/raise :type :not-found
+                  :context {:provider provider}
+                  :hint "provider not configured"))
+      (-> (assoc @cfg :provider provider)
+          (handler request)))))
+
+(defmethod ig/init-key ::handler
+  [_ cfg]
+  (let [cfg (initialize cfg)]
+    {:handler (wrap-handler cfg auth-handler)
+     :callback-handler (wrap-handler cfg callback-handler)}))
+
+(defn- discover-oidc-config
+  [{:keys [base-uri] :as opts}]
+
+  (let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
+        response      (ex/try (http/send! {:method :get :uri (str discovery-uri)}))]
+    (cond
+      (ex/exception? response)
+      (do
+        (l/warn :hint "unable to discover oidc configuration"
+                :discover-uri (str discovery-uri)
+                :cause response)
+        nil)
+
+      (= 200 (:status response))
+      (let [data (json/read-str (:body response))]
+        {:token-uri (get data "token_endpoint")
+         :auth-uri  (get data "authorization_endpoint")
+         :user-uri  (get data "userinfo_endpoint")})
+
+      :else
+      (do
+        (l/warn :hint "unable to discover OIDC configuration"
+                :uri (str discovery-uri)
+                :response-status-code (:status response))
+        nil))))
+
+(defn- obfuscate-string
+  [s]
+  (if (< (count s) 10)
+    (apply str (take (count s) (repeat "*")))
+    (str (subs s 0 5)
+         (apply str (take (- (count s) 5) (repeat "*"))))))
+
+(defn- initialize-oidc-provider
+  [cfg]
+  (let [opts {:base-uri      (cf/get :oidc-base-uri)
+              :client-id     (cf/get :oidc-client-id)
+              :client-secret (cf/get :oidc-client-secret)
+              :token-uri     (cf/get :oidc-token-uri)
+              :auth-uri      (cf/get :oidc-auth-uri)
+              :user-uri      (cf/get :oidc-user-uri)
+              :scopes        (cf/get :oidc-scopes #{"openid" "profile" "email"})
+              :roles-attr    (cf/get :oidc-roles-attr)
+              :roles         (cf/get :oidc-roles)
+              :name          "oidc"}]
+    (if (and (string? (:base-uri opts))
+             (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/debug :hint "initialize oidc provider" :name "generic-oidc"
+                 :opts (update opts :client-secret obfuscate-string))
+        (if (and (string? (:token-uri opts))
+                 (string? (:user-uri opts))
+                 (string? (:auth-uri opts)))
+          (do
+            (l/debug :hint "initialized with user provided configuration")
+            (assoc-in cfg [:providers "oidc"] opts))
+          (do
+            (l/debug :hint "trying to discover oidc provider configuration using BASE_URI")
+            (if-let [opts' (discover-oidc-config opts)]
+              (do
+                (l/debug :hint "discovered opts" :additional-opts opts')
+                (assoc-in cfg [:providers "oidc"] (merge opts opts')))
+
+              cfg))))
+      cfg)))
+
+(defn- initialize-google-provider
+  [cfg]
+  (let [opts {:client-id     (cf/get :google-client-id)
+              :client-secret (cf/get :google-client-secret)
+              :scopes        #{"openid" "email" "profile"}
+              :auth-uri      "https://accounts.google.com/o/oauth2/v2/auth"
+              :token-uri     "https://oauth2.googleapis.com/token"
+              :user-uri      "https://openidconnect.googleapis.com/v1/userinfo"
+              :name          "google"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "google"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "google"] opts))
+      cfg)))
+
+(defn- initialize-github-provider
+  [cfg]
+  (let [opts {:client-id     (cf/get :github-client-id)
+              :client-secret (cf/get :github-client-secret)
+              :scopes        #{"read:user" "user:email"}
+              :auth-uri      "https://github.com/login/oauth/authorize"
+              :token-uri     "https://github.com/login/oauth/access_token"
+              :user-uri      "https://api.github.com/user"
+              :name          "github"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "github"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "github"] opts))
+      cfg)))
+
+
+(defn- initialize-gitlab-provider
+  [cfg]
+  (let [base (cf/get :gitlab-base-uri "https://gitlab.com")
+        opts {:base-uri      base
+              :client-id     (cf/get :gitlab-client-id)
+              :client-secret (cf/get :gitlab-client-secret)
+              :scopes        #{"read_user"}
+              :auth-uri      (str base "/oauth/authorize")
+              :token-uri     (str base "/oauth/token")
+              :user-uri      (str base "/api/v4/user")
+              :name          "gitlab"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "gitlab"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "gitlab"] opts))
+      cfg)))
+
+(defn- initialize
+  [cfg]
+  (let [cfg (agent cfg :error-mode :continue)]
+    (send-off cfg initialize-google-provider)
+    (send-off cfg initialize-gitlab-provider)
+    (send-off cfg initialize-github-provider)
+    (send-off cfg initialize-oidc-provider)
+    cfg))

backend/src/app/http/session.clj

@@ -2,73 +2,203 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.session
   (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.config :as cfg]
    [app.db :as db]
-   [app.util.log4j :refer [update-thread-context!]]
-   [buddy.core.codecs :as bc]
-   [buddy.core.nonce :as bn]
+   [app.metrics :as mtx]
+   [app.util.async :as aa]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
 
-(defn next-session-id
-  ([] (next-session-id 96))
-  ([n]
-   (-> (bn/random-nonce n)
-       (bc/bytes->b64u)
-       (bc/bytes->str))))
+;; A default cookie name for storing the session. We don't allow
+;; configure it.
+(def cookie-name "auth-token")
 
-(defn create!
-  [{:keys [conn] :as cfg} {:keys [profile-id user-agent]}]
-  (let [id (next-session-id)]
-    (db/insert! conn :http-session {:id id
-                                    :profile-id profile-id
-                                    :user-agent user-agent})
-    id))
+;; --- IMPL
 
-(defn delete!
-  [{:keys [conn cookie-name] :as cfg} request]
-  (when-let [token (get-in request [:cookies cookie-name :value])]
+(defn- create-session
+  [{:keys [conn tokens] :as cfg} {:keys [profile-id headers] :as request}]
+  (let [token  (tokens :generate {:iss "authentication"
+                                  :iat (dt/now)
+                                  :uid profile-id})
+        now    (dt/now)
+        params {:user-agent (get headers "user-agent")
+                :profile-id profile-id
+                :created-at now
+                :updated-at now
+                :id token}]
+    (db/insert! conn :http-session params)))
+
+(defn- delete-session
+  [{:keys [conn] :as cfg} {:keys [cookies] :as request}]
+  (when-let [token (get-in cookies [cookie-name :value])]
     (db/delete! conn :http-session {:id token}))
   nil)
 
-(defn retrieve
-  [{:keys [conn] :as cfg} token]
-  (when token
-    (-> (db/exec-one! conn ["select profile_id from http_session where id = ?" token])
-        (:profile-id))))
+(defn- retrieve-session
+  [{:keys [conn] :as cfg} id]
+  (when id
+    (db/exec-one! conn ["select id, profile_id from http_session where id = ?" id])))
 
-(defn retrieve-from-request
-  [{:keys [cookie-name] :as cfg} request]
-  (->> (get-in request [:cookies cookie-name :value])
-       (retrieve cfg)))
+(defn- retrieve-from-request
+  [cfg {:keys [cookies] :as request}]
+  (->> (get-in cookies [cookie-name :value])
+       (retrieve-session cfg)))
 
-(defn cookies
-  [{:keys [cookie-name] :as cfg} vals]
-  {cookie-name (merge vals {:path "/" :http-only true})})
+(defn- add-cookies
+  [response {:keys [id] :as session}]
+  (let [cors?   (contains? cfg/flags :cors)
+        secure? (contains? cfg/flags :secure-session-cookies)]
+    (assoc response :cookies {cookie-name {:path "/"
+                                           :http-only true
+                                           :value id
+                                           :same-site (if cors? :none :lax)
+                                           :secure secure?}})))
 
-(defn middleware
+(defn- clear-cookies
+  [response]
+  (assoc response :cookies {cookie-name {:value "" :max-age -1}}))
+
+(defn- middleware
   [cfg handler]
   (fn [request]
-    (if-let [profile-id (retrieve-from-request cfg request)]
+    (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
       (do
-        (update-thread-context! {:profile-id profile-id})
-        (handler (assoc request :profile-id profile-id)))
+        (a/>!! (::events-ch cfg) id)
+        (l/set-context! {:profile-id profile-id})
+        (handler (assoc request :profile-id profile-id :session-id id)))
       (handler request))))
 
+;; --- STATE INIT: SESSION
+
 (defmethod ig/pre-init-spec ::session [_]
   (s/keys :req-un [::db/pool]))
 
 (defmethod ig/prep-key ::session
   [_ cfg]
-  (merge {:cookie-name "auth-token"} cfg))
+  (d/merge {:buffer-size 128} (d/without-nils cfg)))
 
 (defmethod ig/init-key ::session
   [_ {:keys [pool] :as cfg}]
-  (let [cfg (assoc cfg :conn pool)]
-    (merge cfg {:middleware #(middleware cfg %)})))
+  (let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
+        cfg    (-> cfg
+                   (assoc :conn pool)
+                   (assoc ::events-ch events))]
+    (-> cfg
+        (assoc :middleware #(middleware cfg %))
+        (assoc :create (fn [profile-id]
+                         (fn [request response]
+                           (let [request (assoc request :profile-id profile-id)
+                                 session (create-session cfg request)]
+                             (add-cookies response session)))))
+        (assoc :delete (fn [request response]
+                         (delete-session cfg request)
+                         (-> response
+                             (assoc :status 204)
+                             (assoc :body "")
+                             (clear-cookies)))))))
+
+(defmethod ig/halt-key! ::session
+  [_ data]
+  (a/close! (::events-ch data)))
+
+
+;; --- STATE INIT: SESSION UPDATER
+
+(declare update-sessions)
+
+(s/def ::session map?)
+(s/def ::max-batch-age ::cfg/http-session-updater-batch-max-age)
+(s/def ::max-batch-size ::cfg/http-session-updater-batch-max-size)
+
+(defmethod ig/pre-init-spec ::updater [_]
+  (s/keys :req-un [::db/pool ::wrk/executor ::mtx/metrics ::session]
+          :opt-un [::max-batch-age
+                   ::max-batch-size]))
+
+(defmethod ig/prep-key ::updater
+  [_ cfg]
+  (merge {:max-batch-age (dt/duration {:minutes 5})
+          :max-batch-size 200}
+         (d/without-nils cfg)))
+
+(defmethod ig/init-key ::updater
+  [_ {:keys [session metrics] :as cfg}]
+  (l/info :action "initialize session updater"
+          :max-batch-age (str (:max-batch-age cfg))
+          :max-batch-size (str (:max-batch-size cfg)))
+  (let [input (aa/batch (::events-ch session)
+                        {:max-batch-size (:max-batch-size cfg)
+                         :max-batch-age (inst-ms (:max-batch-age cfg))})
+        mcnt  (mtx/create
+               {:name "http_session_update_total"
+                :help "A counter of session update batch events."
+                :registry (:registry metrics)
+                :type :counter})]
+    (a/go-loop []
+      (when-let [[reason batch] (a/<! input)]
+        (let [result (a/<! (update-sessions cfg batch))]
+          (mcnt :inc)
+          (cond
+            (ex/exception? result)
+            (l/error :task "updater"
+                     :hint "unexpected error on update sessions"
+                     :cause result)
+
+            (= :size reason)
+            (l/debug :task "updater"
+                     :hint "update sessions"
+                     :reason (name reason)
+                     :count result))
+          (recur))))))
+
+(defn- update-sessions
+  [{:keys [pool executor]} ids]
+  (aa/with-thread executor
+    (db/exec-one! pool ["update http_session set updated_at=now() where id = ANY(?)"
+                        (into-array String ids)])
+    (count ids)))
+
+;; --- STATE INIT: SESSION GC
+
+(declare sql:delete-expired)
+
+(s/def ::max-age ::dt/duration)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool]
+          :opt-un [::max-age]))
+
+(defmethod ig/prep-key ::gc-task
+  [_ cfg]
+  (merge {:max-age (dt/duration {:days 15})}
+         (d/without-nils cfg)))
+
+(defmethod ig/init-key ::gc-task
+  [_ {:keys [pool max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
+  (fn [_]
+    (db/with-atomic [conn pool]
+      (let [interval (db/interval max-age)
+            result   (db/exec-one! conn [sql:delete-expired interval interval])
+            result   (:next.jdbc/update-count result)]
+        (l/debug :task "gc"
+                 :hint "clean http sessions"
+                 :deleted result)
+        result))))
+
+(def ^:private
+  sql:delete-expired
+  "delete from http_session
+    where updated_at < now() - ?::interval
+       or (updated_at is null and
+           created_at < now() - ?::interval)")

backend/src/app/http/websocket.clj

@@ -0,0 +1,145 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.websocket
+  "A penpot notification service for file cooperative edition."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.metrics :as mtx]
+   [app.util.websocket :as ws]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [yetti.websocket :as yws]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; WEBSOCKET HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare send-presence!)
+
+(defmulti handle-message
+  (fn [_wsp message] (:type message)))
+
+(defmethod handle-message :connect
+  [wsp _]
+  (let [{:keys [msgbus file-id team-id session-id ::ws/output-ch]} @wsp
+        sub-ch (a/chan (a/dropping-buffer 32))]
+
+    (swap! wsp assoc :sub-ch sub-ch)
+
+    ;; Start a subscription forwarding goroutine
+    (a/go-loop []
+      (when-let [val (a/<! sub-ch)]
+        (when-not (= (:session-id val) session-id)
+          ;; If we receive a connect message of other user, we need
+          ;; to send an update presence to all participants.
+          (when (= :connect (:type val))
+            (a/<! (send-presence! @wsp :presence)))
+
+          ;; Then, just forward the message
+          (a/>! output-ch val))
+        (recur)))
+
+    (a/go
+      (a/<! (msgbus :sub {:topics [file-id team-id] :chan sub-ch}))
+      (a/<! (send-presence! @wsp :connect)))))
+
+(defmethod handle-message :disconnect
+  [wsp _]
+  (a/close! (:sub-ch @wsp))
+  (send-presence! @wsp :disconnect))
+
+(defmethod handle-message :keepalive
+  [_ _]
+  (a/go :nothing))
+
+(defmethod handle-message :pointer-update
+  [wsp message]
+  (let [{:keys [profile-id file-id session-id msgbus]} @wsp]
+    (msgbus :pub {:topic file-id
+                  :message (assoc message
+                                  :profile-id profile-id
+                                  :session-id session-id)})))
+
+(defmethod handle-message :default
+  [_ message]
+  (a/go
+    (l/log :level :warn
+           :msg "received unexpected message"
+           :message message)))
+
+;; --- IMPL
+
+(defn- send-presence!
+  ([ws] (send-presence! ws :presence))
+  ([{:keys [msgbus session-id profile-id file-id]} type]
+   (msgbus :pub {:topic file-id
+                 :message {:type type
+                           :session-id session-id
+                           :profile-id profile-id}})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare retrieve-file)
+
+(s/def ::msgbus fn?)
+(s/def ::file-id ::us/uuid)
+(s/def ::session-id ::us/uuid)
+
+(s/def ::handler-params
+  (s/keys :req-un [::file-id ::session-id]))
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::msgbus ::db/pool ::mtx/metrics ::wrk/executor]))
+
+(defmethod ig/init-key ::handler
+  [_ {:keys [metrics pool] :as cfg}]
+  (let [metrics {:connections (get-in metrics [:definitions :websocket-active-connections])
+                 :messages    (get-in metrics [:definitions :websocket-messages-total])
+                 :sessions    (get-in metrics [:definitions :websocket-session-timing])}]
+    (fn [{:keys [profile-id params] :as req}]
+      (let [params (us/conform ::handler-params params)
+            file   (retrieve-file pool (:file-id params))
+            cfg    (-> (merge cfg params)
+                       (assoc :profile-id profile-id)
+                       (assoc :team-id (:team-id file))
+                       (assoc ::ws/metrics metrics))]
+
+        (when-not profile-id
+          (ex/raise :type :authentication
+                    :hint "Authentication required."))
+
+        (when-not file
+          (ex/raise :type :not-found
+                    :code :object-not-found))
+
+        (when-not (yws/upgrade-request? req)
+          (ex/raise :type :validation
+                    :code :websocket-request-expected
+                    :hint "this endpoint only accepts websocket connections"))
+
+        (->> (ws/handler handle-message cfg)
+             (yws/upgrade req))))))
+
+(def ^:private
+  sql:retrieve-file
+  "select f.id as id,
+          p.team_id as team_id
+     from file as f
+     join project as p on (p.id = f.project_id)
+    where f.id = ?")
+
+(defn- retrieve-file
+  [conn id]
+  (db/exec-one! conn [sql:retrieve-file id]))
+

backend/src/app/loggers/audit.clj

@@ -0,0 +1,325 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.audit
+  "Services related to the user activity (audit log)."
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.common.transit :as t]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]
+   [lambdaisland.uri :as u]
+   [promesa.exec :as px]))
+
+(defn parse-client-ip
+  [{:keys [headers] :as request}]
+  (or (some-> (get headers "x-forwarded-for") (str/split ",") first)
+      (get headers "x-real-ip")
+      (get request :remote-addr)))
+
+(defn profile->props
+  [profile]
+  (-> profile
+      (select-keys [:is-active :is-muted :auth-backend :email :default-team-id :default-project-id :fullname :lang])
+      (merge (:props profile))
+      (d/without-nils)))
+
+(defn clean-props
+  [{:keys [profile-id] :as event}]
+  (letfn [(clean-common [props]
+            (-> props
+                (dissoc :session-id)
+                (dissoc :password)
+                (dissoc :old-password)
+                (dissoc :token)))
+
+          (clean-profile-id [props]
+            (cond-> props
+              (= profile-id (:profile-id props))
+              (dissoc :profile-id)))
+
+          (clean-complex-data [props]
+            (reduce-kv (fn [props k v]
+                         (cond-> props
+                           (or (string? v)
+                               (uuid? v)
+                               (boolean? v)
+                               (number? v))
+                           (assoc k v)
+
+                           (keyword? v)
+                           (assoc k (name v))))
+                       {}
+                       props))]
+
+    (update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP Handler
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare persist-http-events)
+
+(s/def ::profile-id ::us/uuid)
+(s/def ::name ::us/string)
+(s/def ::type ::us/string)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::timestamp dt/instant?)
+(s/def ::context (s/map-of ::us/keyword any?))
+
+(s/def ::event
+  (s/keys :req-un [::type ::name ::props ::timestamp ::profile-id]
+          :opt-un [::context]))
+
+(s/def ::events (s/every ::event))
+
+(defmethod ig/init-key ::http-handler
+  [_  {:keys [executor] :as cfg}]
+  (fn [{:keys [params profile-id] :as request}]
+    (when (contains? cf/flags :audit-log)
+      (let [events  (->> (:events params)
+                         (remove #(not= profile-id (:profile-id %)))
+                         (us/conform ::events))
+            ip-addr (parse-client-ip request)
+            cfg     (-> cfg
+                        (assoc :source "frontend")
+                        (assoc :events events)
+                        (assoc :ip-addr ip-addr))]
+        (px/run! executor #(persist-http-events cfg))))
+    {:status 204 :body ""}))
+
+(defn- persist-http-events
+  [{:keys [pool events ip-addr source] :as cfg}]
+  (try
+    (let [columns    [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context]
+          prepare-xf (map (fn [event]
+                            [(uuid/next)
+                             (:name event)
+                             source
+                             (:type event)
+                             (:timestamp event)
+                             (:profile-id event)
+                             (db/inet ip-addr)
+                             (db/tjson (:props event))
+                             (db/tjson (d/without-nils (:context event)))]))
+          events     (us/conform ::events events)]
+      (when (seq events)
+        (->> (into [] prepare-xf events)
+             (db/insert-multi! pool :audit-log columns))))
+    (catch Throwable e
+      (let [xdata (ex-data e)]
+        (if (= :spec-validation (:code xdata))
+          (l/error ::l/raw (str "spec validation on persist-events:\n"
+                                (:explain xdata)))
+          (l/error :hint "error on persist-events"
+                   :cause e))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Collector
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Defines a service that collects the audit/activity log using
+;; internal database. Later this audit log can be transferred to
+;; an external storage and data cleared.
+
+(declare persist-events)
+
+(defmethod ig/pre-init-spec ::collector [_]
+  (s/keys :req-un [::db/pool ::wrk/executor]))
+
+(def event-xform
+  (comp
+   (filter :profile-id)
+   (map clean-props)))
+
+(defmethod ig/init-key ::collector
+  [_ cfg]
+  (when (contains? cf/flags :audit-log)
+    (l/info :msg "initializing audit log collector")
+    (let [input  (a/chan 512 event-xform)
+          buffer (aa/batch input {:max-batch-size 100
+                                  :max-batch-age (* 10 1000) ; 10s
+                                  :init []})]
+      (a/go-loop []
+        (when-let [[_type events] (a/<! buffer)]
+          (let [res (a/<! (persist-events cfg events))]
+            (when (ex/exception? res)
+              (l/error :hint "error on persisting events"
+                       :cause res)))
+          (recur)))
+
+      (fn [& {:keys [cmd] :as params}]
+        (let [params (-> params
+                         (dissoc :cmd)
+                         (assoc :tracked-at (dt/now)))]
+          (case cmd
+            :stop   (a/close! input)
+            :submit (when-not (a/offer! input params)
+                      (l/warn :msg "activity channel is full"))))))))
+
+
+(defn- persist-events
+  [{:keys [pool executor] :as cfg} events]
+  (letfn [(event->row [event]
+            (when (:profile-id event)
+              [(uuid/next)
+               (:name event)
+               (:type event)
+               (:profile-id event)
+               (:tracked-at event)
+               (some-> (:ip-addr event) db/inet)
+               (db/tjson (:props event))
+               "backend"]))]
+    (aa/with-thread executor
+      (when (seq events)
+        (db/with-atomic [conn pool]
+          (db/insert-multi! conn :audit-log
+                            [:id :name :type :profile-id :tracked-at :ip-addr :props :source]
+                            (sequence (keep event->row) events)))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Archive Task
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; This is a task responsible to send the accumulated events to an
+;; external service for archival.
+
+(declare archive-events)
+
+(s/def ::uri ::us/string)
+(s/def ::tokens fn?)
+
+(defmethod ig/pre-init-spec ::archive-task [_]
+  (s/keys :req-un [::db/pool ::tokens]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::archive-task
+  [_ {:keys [uri] :as cfg}]
+  (fn [props]
+    ;; NOTE: this let allows overwrite default configured values from
+    ;; the repl, when manually invoking the task.
+    (let [enabled (or (contains? cf/flags :audit-log-archive)
+                      (:enabled props false))
+          uri     (or uri (:uri props))
+          cfg     (assoc cfg :uri uri)]
+      (when (and enabled (not uri))
+        (ex/raise :type :internal
+                  :code :task-not-configured
+                  :hint "archive task not configured, missing uri"))
+      (when enabled
+        (loop []
+          (let [res (archive-events cfg)]
+            (when (= res :continue)
+              (aa/thread-sleep 200)
+              (recur))))))))
+
+(def sql:retrieve-batch-of-audit-log
+  "select * from audit_log
+    where archived_at is null
+    order by created_at asc
+    limit 1000
+      for update skip locked;")
+
+(defn archive-events
+  [{:keys [pool uri tokens] :as cfg}]
+  (letfn [(decode-row [{:keys [props ip-addr context] :as row}]
+            (cond-> row
+              (db/pgobject? props)
+              (assoc :props (db/decode-transit-pgobject props))
+
+              (db/pgobject? context)
+              (assoc :context (db/decode-transit-pgobject context))
+
+              (db/pgobject? ip-addr "inet")
+              (assoc :ip-addr (db/decode-inet ip-addr))))
+
+          (row->event [row]
+            (select-keys row [:type
+                              :name
+                              :source
+                              :created-at
+                              :tracked-at
+                              :profile-id
+                              :ip-addr
+                              :props
+                              :context]))
+
+          (send [events]
+            (let [token   (tokens :generate {:iss "authentication"
+                                             :iat (dt/now)
+                                             :uid uuid/zero})
+                  body    (t/encode {:events events})
+                  headers {"content-type" "application/transit+json"
+                           "origin" (cf/get :public-uri)
+                           "cookie" (u/map->query-string {:auth-token token})}
+                  params  {:uri uri
+                           :timeout 6000
+                           :method :post
+                           :headers headers
+                           :body body}
+                  resp    (http/send! params)]
+              (if (= (:status resp) 204)
+                true
+                (do
+                  (l/warn :hint "unable to archive events"
+                          :resp-status (:status resp))
+                  false))))
+
+          (mark-as-archived [conn rows]
+            (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)"
+                                (->> (map :id rows)
+                                     (into-array java.util.UUID)
+                                     (db/create-array conn "uuid"))]))]
+
+    (db/with-atomic [conn pool]
+      (let [rows   (db/exec! conn [sql:retrieve-batch-of-audit-log])
+            xform  (comp (map decode-row)
+                         (map row->event))
+            events (into [] xform rows)]
+        (when-not (empty? events)
+          (l/debug :action "archive-events" :uri uri :events (count events))
+          (when (send events)
+            (mark-as-archived conn rows)
+            :continue))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GC Task
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def sql:clean-archived
+  "delete from audit_log
+    where archived_at is not null
+      and archived_at < now() - ?::interval")
+
+(defn- clean-archived
+  [{:keys [pool max-age]}]
+  (let [interval (db/interval max-age)
+        result   (db/exec-one! pool [sql:clean-archived interval])
+        result   (:next.jdbc/update-count result)]
+    (l/debug :action "clean archived audit log" :removed result)
+    result))
+
+(s/def ::max-age ::cf/audit-log-gc-max-age)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool ::max-age]))
+
+(defmethod ig/init-key ::gc-task
+  [_ cfg]
+  (fn [_]
+    (clean-archived cfg)))

backend/src/app/loggers/database.clj

@@ -0,0 +1,91 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.database
+  "A specific logger impl that persists errors on the database."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare handle-event)
+
+(defonce enabled (atom true))
+
+(defn- persist-on-database!
+  [{:keys [pool]} {:keys [id] :as event}]
+  (db/insert! pool :server-error-report {:id id :content (db/tjson event)}))
+
+(defn- parse-event-data
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   event))
+
+(defn parse-event
+  [event]
+  (-> (parse-event-data event)
+      (assoc :tenant (cf/get :tenant))
+      (assoc :host (cf/get :host))
+      (assoc :public-uri (cf/get :public-uri))
+      (assoc :version (:full cf/version))
+      (update :id #(or % (uuid/next)))))
+
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [event (parse-event event)
+            uri   (cf/get :public-uri)]
+
+        (l/debug :hint "registering error on database" :id (:id event)
+                 :uri (str uri "/dbg/error/" (:id event)))
+
+        (persist-on-database! cfg event))
+      (catch Exception cause
+        (l/warn :hint "unexpected exception on database error logger" :cause cause)))))
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
+
+(defn error-event?
+  [event]
+  (= "error" (:logger/level event)))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver] :as cfg}]
+  (l/info :msg "initializing database error persistence")
+  (let [output (a/chan (a/sliding-buffer 5) (filter error-event?))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [msg (a/<! output)]
+        (if (nil? msg)
+          (l/info :msg "stoping error reporting loop")
+          (do
+            (a/<! (handle-event cfg msg))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (a/close! output))

backend/src/app/loggers/loki.clj

@@ -0,0 +1,99 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.loki
+  "A Loki integration."
+  (:require
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.config :as cfg]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.json :as json]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(declare handle-event)
+
+(s/def ::uri ::us/string)
+(s/def ::receiver fn?)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::receiver]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver uri] :as cfg}]
+  (when uri
+    (l/info :msg "initializing loki reporter" :uri uri)
+    (let [input (a/chan (a/dropping-buffer 512))]
+      (receiver :sub input)
+      (a/go-loop []
+        (let [msg (a/<! input)]
+          (if (nil? msg)
+            (l/info :msg "stoping error reporting loop")
+            (do
+              (a/<! (handle-event cfg msg))
+              (recur)))))
+      input)))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))
+
+(defn- prepare-payload
+  [event]
+  (let [labels {:host    (cfg/get :host)
+                :tenant  (cfg/get :tenant)
+                :version (:full cfg/version)
+                :logger  (:logger event)
+                :level   (:level event)}]
+    {:streams
+     [{:stream labels
+       :values [[(str (* (inst-ms (:created-at event)) 1000000))
+                 (str (:message event)
+                      (when-let [error (:error event)]
+                        (str "\n" (:trace error))))]]}]}))
+
+(defn- send-log
+  [uri payload i]
+  (try
+    (let [response (http/send! {:uri uri
+                                :timeout 6000
+                                :method :post
+                                :headers {"content-type" "application/json"}
+                                :body (json/write payload)})]
+      (cond
+        (= (:status response) 204)
+        true
+
+        (= (:status response) 400)
+        (do
+          (l/error :hint "error on sending log to loki (no retry)"
+                   :rsp (pr-str response))
+          true)
+
+        :else
+        (do
+          (l/error :hint "error on sending log to loki" :try i
+                   :rsp (pr-str response))
+          false)))
+    (catch Exception e
+      (l/error :hint "error on sending message to loki" :cause e :try i)
+      false)))
+
+(defn- handle-event
+  [{:keys [executor uri]} event]
+  (aa/with-thread executor
+    (let [payload (prepare-payload event)]
+      (loop [i 1]
+        (when (and (not (send-log uri payload i)) (< i 20))
+          (Thread/sleep (* i 2000))
+          (recur (inc i)))))))
+

backend/src/app/loggers/mattermost.clj

@@ -0,0 +1,80 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.mattermost
+  "A mattermost integration for error reporting."
+  (:require
+   [app.common.logging :as l]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.loggers.database :as ldb]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.json :as json]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(defonce enabled (atom true))
+
+(defn- send-mattermost-notification!
+  [cfg {:keys [host id public-uri] :as event}]
+  (try
+    (let [uri  (:uri cfg)
+          text (str "Exception on (host: " host ", url: " public-uri "/dbg/error/" id ")\n"
+                    (when-let [pid (:profile-id event)]
+                      (str "- profile-id: #uuid-" pid "\n")))
+          rsp  (http/send! {:uri uri
+                            :method :post
+                            :headers {"content-type" "application/json"}
+                            :body (json/write-str {:text text})})]
+      (when (not= (:status rsp) 200)
+        (l/error :hint "error on sending data to mattermost"
+                 :response (pr-str rsp))))
+
+    (catch Exception e
+      (l/error :hint "unexpected exception on error reporter"
+               :cause e))))
+
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [event (ldb/parse-event event)]
+        (when @enabled
+          (send-mattermost-notification! cfg event)))
+      (catch Exception e
+        (l/warn :hint "unexpected exception on error reporter" :cause e)))))
+
+
+(s/def ::uri ::cf/error-report-webhook)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver uri] :as cfg}]
+  (when uri
+    (l/info :msg "initializing mattermost error reporter" :uri uri)
+    (let [output (a/chan (a/sliding-buffer 128)
+                         (filter (fn [event]
+                                   (= (:logger/level event) "error"))))]
+      (receiver :sub output)
+      (a/go-loop []
+        (let [msg (a/<! output)]
+          (if (nil? msg)
+            (l/info :msg "stoping error reporting loop")
+            (do
+              (a/<! (handle-event cfg msg))
+              (recur)))))
+      output)))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))

backend/src/app/loggers/sentry.clj

@@ -0,0 +1,170 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.sentry
+  "A mattermost integration for error reporting."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig])
+  (:import
+   io.sentry.Scope
+   io.sentry.IHub
+   io.sentry.Hub
+   io.sentry.NoOpHub
+   io.sentry.protocol.User
+   io.sentry.SentryOptions
+   io.sentry.SentryLevel
+   io.sentry.ScopeCallback))
+
+(defonce enabled (atom true))
+
+(defn- parse-context
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   (:context event)))
+
+(defn- parse-event
+  [event]
+  (assoc event :context (parse-context event)))
+
+(defn- build-sentry-options
+  [cfg]
+  (let [version (:base cf/version)]
+    (doto (SentryOptions.)
+      (.setDebug (:debug cfg false))
+      (.setTracesSampleRate (:traces-sample-rate cfg 1.0))
+      (.setDsn (:dsn cfg))
+      (.setServerName (cf/get :host))
+      (.setEnvironment (cf/get :tenant))
+      (.setAttachServerName true)
+      (.setAttachStacktrace (:attach-stack-trace cfg false))
+      (.setRelease (str "backend@" (if (= version "0.0.0") "develop" version))))))
+
+(defn handle-event
+  [^IHub shub event]
+  (letfn [(set-user! [^Scope scope {:keys [context] :as event}]
+            (let [user (User.)]
+              (.setIpAddress ^User user ^String (:ip-addr context))
+              (when-let [pid (:profile-id context)]
+                (.setId ^User user ^String (str pid)))
+              (.setUser scope ^User user)))
+
+          (set-level! [^Scope scope]
+            (.setLevel scope SentryLevel/ERROR))
+
+          (set-context! [^Scope scope {:keys [context] :as event}]
+            (let [uri (str (cf/get :public-uri) "/dbg/error-by-id/" (:id context))]
+              (.setContexts scope "detailed_error_uri" ^String uri))
+            (when-let [vers (:frontend-version event)]
+              (.setContexts scope "frontend_version" ^String vers))
+            (when-let [puri (:public-uri event)]
+              (.setContexts scope "public_uri" ^String (str puri)))
+            (when-let [uagent (:user-agent context)]
+              (.setContexts scope "user_agent" ^String uagent))
+            (when-let [tenant (:tenant event)]
+              (.setTag scope "tenant" ^String tenant))
+            (when-let [type (:error-type context)]
+              (.setTag scope "error_type" ^String  (str type)))
+            (when-let [code (:error-code context)]
+              (.setTag scope "error_code" ^String (str code)))
+            )
+
+          (capture [^Scope scope {:keys [context error] :as event}]
+            (let [msg   (str (:message error) "\n\n"
+
+                             "======================================================\n"
+                             "=================== Params ===========================\n"
+                             "======================================================\n"
+
+                             (:params context) "\n"
+
+                             (when (:explain context)
+                               (str "======================================================\n"
+                                    "=================== Explain ==========================\n"
+                                    "======================================================\n"
+                                    (:explain context) "\n"))
+
+                             (when (:data context)
+                               (str "======================================================\n"
+                                    "=================== Error Data =======================\n"
+                                    "======================================================\n"
+                                    (:data context) "\n"))
+
+                             (str "======================================================\n"
+                                  "=================== Stack Trace ======================\n"
+                                  "======================================================\n"
+                                  (:trace error))
+
+                             "\n")]
+              (set-user! scope event)
+              (set-level! scope)
+              (set-context! scope event)
+              (.captureMessage ^IHub shub msg)
+              ))
+          ]
+    (when @enabled
+      (.withScope ^IHub shub (reify ScopeCallback
+                               (run [_ scope]
+                                 (->> event
+                                      (parse-event)
+                                      (capture scope))))))
+
+    ))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::receiver any?)
+(s/def ::dsn ::cf/sentry-dsn)
+(s/def ::trace-sample-rate ::cf/sentry-trace-sample-rate)
+(s/def ::attach-stack-trace ::cf/sentry-attach-stack-trace)
+(s/def ::debug ::cf/sentry-debug)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+          :opt-un [::dsn ::trace-sample-rate ::attach-stack-trace]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver dsn executor] :as cfg}]
+  (l/info :msg "initializing sentry reporter" :dsn dsn)
+  (let [opts   (build-sentry-options cfg)
+        shub   (if dsn
+                 (Hub. ^SentryOptions opts)
+                 (NoOpHub/getInstance))
+        output (a/chan (a/sliding-buffer 128)
+                       (filter #(= (:level %) "error")))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [event (a/<! output)]
+        (if (nil? event)
+          (do
+            (l/info :msg "stoping error reporting loop")
+            (.close ^IHub shub))
+          (do
+            (a/<! (aa/with-thread executor (handle-event shub event)))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))

backend/src/app/loggers/zmq.clj

@@ -0,0 +1,106 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.zmq
+  "A generic ZMQ listener."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.util.json :as json]
+   [app.util.time :as dt]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig])
+  (:import
+   org.zeromq.SocketType
+   org.zeromq.ZMQ$Socket
+   org.zeromq.ZContext))
+
+(declare prepare)
+(declare start-rcv-loop)
+
+(s/def ::endpoint ::us/string)
+
+(defmethod ig/pre-init-spec ::receiver [_]
+  (s/keys :opt-un [::endpoint]))
+
+(defmethod ig/init-key ::receiver
+  [_ {:keys [endpoint] :as cfg}]
+  (l/info :msg "initializing ZMQ receiver" :bind endpoint)
+  (let [buffer (a/chan 1)
+        output (a/chan 1 (comp (filter map?)
+                               (keep prepare)))
+        mult   (a/mult output)]
+    (when endpoint
+      (a/thread (start-rcv-loop {:out buffer :endpoint endpoint})))
+    (a/pipe buffer output)
+    (with-meta
+      (fn [cmd ch]
+        (case cmd
+          :sub (a/tap mult ch)
+          :unsub (a/untap mult ch))
+        ch)
+      {::output output
+       ::buffer buffer
+       ::mult mult})))
+
+(defmethod ig/halt-key! ::receiver
+  [_ f]
+  (a/close! (::buffer (meta f))))
+
+(def ^:private json-mapper
+  (json/mapper
+   {:encode-key-fn str/camel
+    :decode-key-fn (comp keyword str/kebab)}))
+
+(defn- start-rcv-loop
+  ([] (start-rcv-loop nil))
+  ([{:keys [out endpoint] :or {endpoint "tcp://localhost:5556"}}]
+   (let [out    (or out (a/chan 1))
+         zctx   (ZContext.)
+         socket (.. zctx (createSocket SocketType/SUB))]
+     (.. socket (connect ^String endpoint))
+     (.. socket (subscribe ""))
+     (.. socket (setReceiveTimeOut 5000))
+     (loop []
+       (let [msg (.recv ^ZMQ$Socket socket)
+             msg (ex/ignoring (json/read msg json-mapper))
+             msg (if (nil? msg) :empty msg)]
+         (if (a/>!! out msg)
+           (recur)
+           (do
+             (.close ^java.lang.AutoCloseable socket)
+             (.close ^java.lang.AutoCloseable zctx))))))))
+
+(s/def ::logger-name string?)
+(s/def ::level string?)
+(s/def ::thread string?)
+(s/def ::time-millis integer?)
+(s/def ::message string?)
+(s/def ::context-map map?)
+(s/def ::throw map?)
+
+(s/def ::log4j-event
+  (s/keys :req-un [::logger-name ::level ::thread ::time-millis ::message]
+          :opt-un [::context-map ::thrown]))
+
+(defn- prepare
+  [event]
+  (if (s/valid? ::log4j-event event)
+    (merge {:message      (:message event)
+            :created-at   (dt/instant (:time-millis event))
+            :logger/name  (:logger-name event)
+            :logger/level (str/lower (:level event))}
+
+           (when-let [thrown (:thrown event)]
+             {:trace (:extended-stack-trace thrown)})
+
+           (:context-map event))
+    (do
+      (l/warn :hint "invalid event" :event event)
+      nil)))

backend/src/app/main.clj

@@ -2,339 +2,350 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.main
   (:require
-   [app.common.data :as d]
-   [app.config :as cfg]
+   [app.common.logging :as l]
+   [app.config :as cf]
    [app.util.time :as dt]
-   [clojure.pprint :as pprint]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]))
+   [integrant.core :as ig])
+  (:gen-class))
 
-;; Set value for all new threads bindings.
-(alter-var-root #'*assert* (constantly (:asserts-enabled cfg/config)))
+(def system-config
+  {:app.db/pool
+   {:uri        (cf/get :database-uri)
+    :username   (cf/get :database-username)
+    :password   (cf/get :database-password)
+    :metrics    (ig/ref :app.metrics/metrics)
+    :migrations (ig/ref :app.migrations/all)
+    :name :main
+    :min-pool-size 0
+    :max-pool-size 60}
 
-(derive :app.telemetry/server :app.http/server)
+   :app.migrations/migrations
+   {}
 
-;; --- Entry point
+   :app.metrics/metrics
+   {}
 
-(defn build-system-config
-  [config]
-  (d/deep-merge
-   {:app.db/pool
-    {:uri        (:database-uri config)
-     :username   (:database-username config)
-     :password   (:database-password config)
-     :metrics    (ig/ref :app.metrics/metrics)
-     :migrations (ig/ref :app.migrations/all)
-     :name "main"
-     :min-pool-size 0
-     :max-pool-size 20}
+   :app.migrations/all
+   {:main (ig/ref :app.migrations/migrations)}
 
-    :app.metrics/metrics
-    {}
 
-    :app.migrations/all
-    {:main (ig/ref :app.migrations/migrations)
-     :telemetry  (ig/ref :app.telemetry/migrations)}
+   :app.msgbus/msgbus
+   {:backend   (cf/get :msgbus-backend :redis)
+    :redis-uri (cf/get :redis-uri)}
 
-    :app.migrations/migrations
-    {}
+   :app.tokens/tokens
+   {:keys (ig/ref :app.setup/keys)}
 
-    :app.telemetry/migrations
-    {}
+   :app.storage/gc-deleted-task
+   {:pool     (ig/ref :app.db/pool)
+    :storage  (ig/ref :app.storage/storage)
+    :min-age  (dt/duration {:hours 2})}
 
-    :app.redis/redis
-    {:uri (:redis-uri config)}
+   :app.storage/gc-touched-task
+   {:pool     (ig/ref :app.db/pool)}
 
-    :app.tokens/tokens
-    {:sprops (ig/ref :app.sprops/props)}
+   :app.storage/recheck-task
+   {:pool     (ig/ref :app.db/pool)
+    :storage  (ig/ref :app.storage/storage)}
 
-    :app.storage/gc-deleted-task
-    {:pool     (ig/ref :app.db/pool)
-     :storage  (ig/ref :app.storage/storage)
-     :min-age  (dt/duration {:hours 2})}
+   :app.http.session/session
+   {:pool   (ig/ref :app.db/pool)
+    :tokens (ig/ref :app.tokens/tokens)}
 
-    :app.storage/gc-touched-task
-    {:pool     (ig/ref :app.db/pool)}
+   :app.http.session/gc-task
+   {:pool        (ig/ref :app.db/pool)
+    :max-age     (cf/get :http-session-idle-max-age)}
 
-    :app.storage/recheck-task
-    {:pool     (ig/ref :app.db/pool)
-     :storage  (ig/ref :app.storage/storage)}
+   :app.http.session/updater
+   {:pool           (ig/ref :app.db/pool)
+    :metrics        (ig/ref :app.metrics/metrics)
+    :executor       (ig/ref :app.worker/executor)
+    :session        (ig/ref :app.http.session/session)
+    :max-batch-age  (cf/get :http-session-updater-batch-max-age)
+    :max-batch-size (cf/get :http-session-updater-batch-max-size)}
 
-    :app.http.session/session
-    {:pool        (ig/ref :app.db/pool)
-     :cookie-name "auth-token"}
+   :app.http.awsns/handler
+   {:tokens  (ig/ref :app.tokens/tokens)
+    :pool    (ig/ref :app.db/pool)}
 
-    :app.http/server
-    {:port    (:http-server-port config)
-     :handler (ig/ref :app.http/router)
-     :metrics (ig/ref :app.metrics/metrics)
-     :ws      {"/ws/notifications" (ig/ref :app.notifications/handler)}}
+   :app.http/server
+   {:port    (cf/get :http-server-port)
+    :host    (cf/get :http-server-host)
+    :router  (ig/ref :app.http/router)
+    :metrics (ig/ref :app.metrics/metrics)}
 
-    :app.http/router
-    {:rpc         (ig/ref :app.rpc/rpc)
-     :session     (ig/ref :app.http.session/session)
-     :tokens      (ig/ref :app.tokens/tokens)
-     :public-uri  (:public-uri config)
-     :metrics     (ig/ref :app.metrics/metrics)
-     :google-auth (ig/ref :app.http.auth/google)
-     :gitlab-auth (ig/ref :app.http.auth/gitlab)
-     :github-auth (ig/ref :app.http.auth/github)
-     :ldap-auth   (ig/ref :app.http.auth/ldap)
-     :assets      (ig/ref :app.http.assets/handlers)
-     :svgparse    (ig/ref :app.svgparse/handler)
-     :storage     (ig/ref :app.storage/storage)
-     :error-report-handler (ig/ref :app.error-reporter/handler)}
+   :app.http/router
+   {:assets               (ig/ref :app.http.assets/handlers)
+    :feedback             (ig/ref :app.http.feedback/handler)
+    :session              (ig/ref :app.http.session/session)
+    :sns-webhook          (ig/ref :app.http.awsns/handler)
+    :oauth                (ig/ref :app.http.oauth/handler)
+    :debug                (ig/ref :app.http.debug/handlers)
+    :ws                   (ig/ref :app.http.websocket/handler)
+    :metrics              (ig/ref :app.metrics/metrics)
+    :public-uri           (cf/get :public-uri)
+    :storage              (ig/ref :app.storage/storage)
+    :tokens               (ig/ref :app.tokens/tokens)
+    :audit-http-handler   (ig/ref :app.loggers.audit/http-handler)
+    :rpc                  (ig/ref :app.rpc/rpc)}
 
-    :app.http.assets/handlers
-    {:metrics           (ig/ref :app.metrics/metrics)
-     :assets-path       (:assets-path config)
-     :storage           (ig/ref :app.storage/storage)
-     :cache-max-age     (dt/duration {:hours 24})
-     :signature-max-age (dt/duration {:hours 24 :minutes 5})}
+   :app.http.debug/handlers
+   {:pool (ig/ref :app.db/pool)}
 
-    :app.http.auth/google
-    {:rpc           (ig/ref :app.rpc/rpc)
-     :session       (ig/ref :app.http.session/session)
-     :tokens        (ig/ref :app.tokens/tokens)
-     :public-uri    (:public-uri config)
-     :client-id     (:google-client-id config)
-     :client-secret (:google-client-secret config)}
+   :app.http.websocket/handler
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :msgbus   (ig/ref :app.msgbus/msgbus)}
 
-    :app.http.auth/github
-    {:rpc           (ig/ref :app.rpc/rpc)
-     :session       (ig/ref :app.http.session/session)
-     :tokens        (ig/ref :app.tokens/tokens)
-     :public-uri    (:public-uri config)
-     :client-id     (:github-client-id config)
-     :client-secret (:github-client-secret config)}
+   :app.http.assets/handlers
+   {:metrics           (ig/ref :app.metrics/metrics)
+    :assets-path       (cf/get :assets-path)
+    :storage           (ig/ref :app.storage/storage)
+    :cache-max-age     (dt/duration {:hours 24})
+    :signature-max-age (dt/duration {:hours 24 :minutes 5})}
 
-    :app.http.auth/gitlab
-    {:rpc           (ig/ref :app.rpc/rpc)
-     :session       (ig/ref :app.http.session/session)
-     :tokens        (ig/ref :app.tokens/tokens)
-     :public-uri    (:public-uri config)
-     :base-uri      (:gitlab-base-uri config)
-     :client-id     (:gitlab-client-id config)
-     :client-secret (:gitlab-client-secret config)}
+   :app.http.feedback/handler
+   {:pool (ig/ref :app.db/pool)}
 
-    :app.http.auth/ldap
-    {:host               (:ldap-auth-host config)
-     :port               (:ldap-auth-port config)
-     :ssl                (:ldap-auth-ssl config)
-     :starttls           (:ldap-auth-starttls config)
-     :user-query         (:ldap-auth-user-query config)
-     :username-attribute (:ldap-auth-username-attribute config)
-     :email-attribute    (:ldap-auth-email-attribute config)
-     :fullname-attribute (:ldap-auth-fullname-attribute config)
-     :avatar-attribute   (:ldap-auth-avatar-attribute config)
-     :base-dn            (:ldap-auth-base-dn config)
-     :session            (ig/ref :app.http.session/session)
-     :rpc                (ig/ref :app.rpc/rpc)}
+   :app.http.oauth/handler
+   {:rpc        (ig/ref :app.rpc/rpc)
+    :session    (ig/ref :app.http.session/session)
+    :pool       (ig/ref :app.db/pool)
+    :tokens     (ig/ref :app.tokens/tokens)
+    :audit      (ig/ref :app.loggers.audit/collector)
+    :public-uri (cf/get :public-uri)}
 
-    :app.svgparse/svgc
-    {:metrics (ig/ref :app.metrics/metrics)}
+   :app.rpc/rpc
+   {:pool       (ig/ref :app.db/pool)
+    :session    (ig/ref :app.http.session/session)
+    :tokens     (ig/ref :app.tokens/tokens)
+    :metrics    (ig/ref :app.metrics/metrics)
+    :storage    (ig/ref :app.storage/storage)
+    :msgbus     (ig/ref :app.msgbus/msgbus)
+    :public-uri (cf/get :public-uri)
+    :audit      (ig/ref :app.loggers.audit/collector)}
 
-    ;; HTTP Handler for SVG parsing
-    :app.svgparse/handler
-    {:metrics (ig/ref :app.metrics/metrics)
-     :svgc    (ig/ref :app.svgparse/svgc)}
+   :app.worker/executor
+   {:min-threads 0
+    :max-threads 256
+    :idle-timeout 60000
+    :name :worker}
 
-    ;; RLimit definition for password hashing
-    :app.rlimits/password
-    (:rlimits-password config)
+   :app.worker/worker
+   {:executor (ig/ref :app.worker/executor)
+    :tasks    (ig/ref :app.worker/registry)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :pool     (ig/ref :app.db/pool)}
 
-    ;; RLimit definition for image processing
-    :app.rlimits/image
-    (:rlimits-image config)
+   :app.worker/scheduler
+   {:executor   (ig/ref :app.worker/executor)
+    :tasks      (ig/ref :app.worker/registry)
+    :pool       (ig/ref :app.db/pool)
+    :schedule
+    [{:cron #app/cron "0 0 0 * * ?" ;; daily
+      :task :file-media-gc}
 
-    ;; A collection of rlimits as hash-map.
-    :app.rlimits/all
-    {:password (ig/ref :app.rlimits/password)
-     :image    (ig/ref :app.rlimits/image)}
+     {:cron #app/cron "0 0 * * * ?"  ;; hourly
+      :task :file-xlog-gc}
 
-    :app.rpc/rpc
-    {:pool    (ig/ref :app.db/pool)
-     :session (ig/ref :app.http.session/session)
-     :tokens  (ig/ref :app.tokens/tokens)
-     :metrics (ig/ref :app.metrics/metrics)
-     :storage (ig/ref :app.storage/storage)
-     :redis   (ig/ref :app.redis/redis)
-     :rlimits (ig/ref :app.rlimits/all)
-     :svgc    (ig/ref :app.svgparse/svgc)}
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :storage-deleted-gc}
 
-    :app.notifications/handler
-    {:redis   (ig/ref :app.redis/redis)
-     :pool    (ig/ref :app.db/pool)
-     :session (ig/ref :app.http.session/session)
-     :metrics (ig/ref :app.metrics/metrics)}
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :storage-touched-gc}
 
-    :app.worker/executor
-    {:name "worker"}
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :session-gc}
 
-    :app.worker/worker
-    {:executor   (ig/ref :app.worker/executor)
-     :pool       (ig/ref :app.db/pool)
-     :tasks      (ig/ref :app.tasks/all)}
+     {:cron #app/cron "0 0 * * * ?"  ;; hourly
+      :task :storage-recheck}
 
-    :app.worker/scheduler
-    {:executor   (ig/ref :app.worker/executor)
-     :pool       (ig/ref :app.db/pool)
-     :schedule
-     [{:id "file-media-gc"
-       :cron #app/cron "0 0 0 */1 * ? *" ;; daily
-       :fn (ig/ref :app.tasks.file-media-gc/handler)}
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :objects-gc}
 
-      {:id "file-xlog-gc"
-       :cron #app/cron "0 0 */6 * * ?"  ;; every 2 hours
-       :fn (ig/ref :app.tasks.file-xlog-gc/handler)}
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :tasks-gc}
 
-      {:id "storage-deleted-gc"
-       :cron #app/cron "0 0 */6 * * ?"  ;; every 6 hours
-       :fn (ig/ref :app.storage/gc-deleted-task)}
+     (when (cf/get :fdata-storage-backed)
+       {:cron #app/cron "0 0 * * * ?"  ;; hourly
+        :task :file-offload})
 
-      {:id "storage-touched-gc"
-       :cron #app/cron "0 30 */6 * * ?"  ;; every 6 hours
-       :fn (ig/ref :app.storage/gc-touched-task)}
+     (when (contains? cf/flags :audit-log-archive)
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
+        :task :audit-log-archive})
 
-      {:id "storage-recheck"
-       :cron #app/cron "0 0 */6 * * ?"  ;; every 6 hours
-       :fn (ig/ref :app.storage/recheck-task)}
+     (when (contains? cf/flags :audit-log-gc)
+       {:cron #app/cron "0 0 0 * * ?" ;; daily
+        :task :audit-log-gc})
 
-      {:id "tasks-gc"
-       :cron #app/cron "0 0 0 */1 * ?"  ;; daily
-       :fn (ig/ref :app.tasks.tasks-gc/handler)}
+     (when (or (contains? cf/flags :telemetry)
+               (cf/get :telemetry-enabled))
+       {:cron #app/cron "0 30 */3,23 * * ?"
+        :task :telemetry})]}
 
-      (when (:telemetry-enabled config)
-        {:id   "telemetry"
-         :cron #app/cron "0 0 */6 * * ?" ;; every 6h
-         :uri  (:telemetry-uri config)
-         :fn   (ig/ref :app.tasks.telemetry/handler)})]}
+   :app.worker/registry
+   {:metrics (ig/ref :app.metrics/metrics)
+    :tasks
+    {:sendmail           (ig/ref :app.emails/sendmail-handler)
+     :objects-gc         (ig/ref :app.tasks.objects-gc/handler)
+     :file-media-gc      (ig/ref :app.tasks.file-media-gc/handler)
+     :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
+     :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
+     :storage-touched-gc (ig/ref :app.storage/gc-touched-task)
+     :storage-recheck    (ig/ref :app.storage/recheck-task)
+     :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
+     :telemetry          (ig/ref :app.tasks.telemetry/handler)
+     :session-gc         (ig/ref :app.http.session/gc-task)
+     :file-offload       (ig/ref :app.tasks.file-offload/handler)
+     :audit-log-archive  (ig/ref :app.loggers.audit/archive-task)
+     :audit-log-gc       (ig/ref :app.loggers.audit/gc-task)}}
 
-    :app.tasks/all
-    {"sendmail"       (ig/ref :app.tasks.sendmail/handler)
-     "delete-object"  (ig/ref :app.tasks.delete-object/handler)
-     "delete-profile" (ig/ref :app.tasks.delete-profile/handler)}
+   :app.emails/sendmail-handler
+   {:host             (cf/get :smtp-host)
+    :port             (cf/get :smtp-port)
+    :ssl              (cf/get :smtp-ssl)
+    :tls              (cf/get :smtp-tls)
+    :username         (cf/get :smtp-username)
+    :password         (cf/get :smtp-password)
+    :metrics          (ig/ref :app.metrics/metrics)
+    :default-reply-to (cf/get :smtp-default-reply-to)
+    :default-from     (cf/get :smtp-default-from)}
 
-    :app.tasks.sendmail/handler
-    {:host             (:smtp-host config)
-     :port             (:smtp-port config)
-     :ssl              (:smtp-ssl config)
-     :tls              (:smtp-tls config)
-     :enabled          (:smtp-enabled config)
-     :username         (:smtp-username config)
-     :password         (:smtp-password config)
-     :metrics          (ig/ref :app.metrics/metrics)
-     :default-reply-to (:smtp-default-reply-to config)
-     :default-from     (:smtp-default-from config)}
+   :app.tasks.tasks-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age cf/deletion-delay}
 
-    :app.tasks.tasks-gc/handler
-    {:pool    (ig/ref :app.db/pool)
-     :max-age (dt/duration {:hours 24})
-     :metrics (ig/ref :app.metrics/metrics)}
+   :app.tasks.objects-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :storage (ig/ref :app.storage/storage)
+    :max-age cf/deletion-delay}
 
-    :app.tasks.delete-object/handler
-    {:pool    (ig/ref :app.db/pool)
-     :metrics (ig/ref :app.metrics/metrics)}
+   :app.tasks.file-media-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age cf/deletion-delay}
 
-    :app.tasks.delete-storage-object/handler
-    {:pool    (ig/ref :app.db/pool)
-     :storage (ig/ref :app.storage/storage)
-     :metrics (ig/ref :app.metrics/metrics)}
+   :app.tasks.file-xlog-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age (dt/duration {:hours 72})}
 
-    :app.tasks.delete-profile/handler
-    {:pool    (ig/ref :app.db/pool)
-     :metrics (ig/ref :app.metrics/metrics)}
+   :app.tasks.file-offload/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age (dt/duration {:seconds 5})
+    :storage (ig/ref :app.storage/storage)
+    :backend (cf/get :fdata-storage-backed :fdata-s3)}
 
-    :app.tasks.file-media-gc/handler
-    {:pool    (ig/ref :app.db/pool)
-     :metrics (ig/ref :app.metrics/metrics)
-     :max-age (dt/duration {:hours 48})}
+   :app.tasks.telemetry/handler
+   {:pool        (ig/ref :app.db/pool)
+    :version     (:full cf/version)
+    :uri         (cf/get :telemetry-uri)
+    :sprops      (ig/ref :app.setup/props)}
 
-    :app.tasks.file-xlog-gc/handler
-    {:pool    (ig/ref :app.db/pool)
-     :metrics (ig/ref :app.metrics/metrics)
-     :max-age (dt/duration {:hours 24})}
+   :app.srepl/server
+   {:port (cf/get :srepl-port)
+    :host (cf/get :srepl-host)}
 
-    :app.tasks.telemetry/handler
-    {:pool        (ig/ref :app.db/pool)
-     :version     (:full cfg/version)
-     :uri         (:telemetry-uri config)
-     :sprops      (ig/ref :app.sprops/props)}
+   :app.setup/props
+   {:pool (ig/ref :app.db/pool)
+    :key  (cf/get :secret-key)}
 
-    :app.srepl/server
-    {:port (:srepl-port config)
-     :host (:srepl-host config)}
+   :app.setup/keys
+   {:props (ig/ref :app.setup/props)}
 
-    :app.sprops/props
-    {:pool (ig/ref :app.db/pool)}
+   :app.loggers.zmq/receiver
+   {:endpoint (cf/get :loggers-zmq-uri)}
 
-    :app.error-reporter/reporter
-    {:uri      (:error-report-webhook config)
-     :pool     (ig/ref :app.db/pool)
-     :executor (ig/ref :app.worker/executor)}
+   :app.loggers.audit/http-handler
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
 
-    :app.error-reporter/handler
-    {:pool (ig/ref :app.db/pool)}
+   :app.loggers.audit/collector
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
 
-    :app.storage/storage
-    {:pool     (ig/ref :app.db/pool)
-     :executor (ig/ref :app.worker/executor)
-     :backend  (:storage-backend config :fs)
-     :backends {:s3  (ig/ref [::main :app.storage.s3/backend])
-                :db  (ig/ref [::main :app.storage.db/backend])
-                :fs  (ig/ref [::main :app.storage.fs/backend])
-                :tmp (ig/ref [::tmp  :app.storage.fs/backend])}}
+   :app.loggers.audit/archive-task
+   {:uri      (cf/get :audit-log-archive-uri)
+    :tokens   (ig/ref :app.tokens/tokens)
+    :pool     (ig/ref :app.db/pool)}
 
-    [::main :app.storage.s3/backend]
-    {:region (:storage-s3-region config)
-     :bucket (:storage-s3-bucket config)}
+   :app.loggers.audit/gc-task
+   {:max-age  (cf/get :audit-log-gc-max-age cf/deletion-delay)
+    :pool     (ig/ref :app.db/pool)}
 
-    [::main :app.storage.fs/backend]
-    {:directory (:storage-fs-directory config)}
+   :app.loggers.loki/reporter
+   {:uri      (cf/get :loggers-loki-uri)
+    :receiver (ig/ref :app.loggers.zmq/receiver)
+    :executor (ig/ref :app.worker/executor)}
 
-    [::tmp :app.storage.fs/backend]
-    {:directory "/tmp/penpot"}
+   :app.loggers.mattermost/reporter
+   {:uri      (cf/get :error-report-webhook)
+    :receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
 
-    [::main :app.storage.db/backend]
-    {:pool (ig/ref :app.db/pool)}}
+   :app.loggers.database/reporter
+   {:receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
 
-   (when (:telemetry-server-enabled config)
-     {:app.telemetry/handler
-      {:pool     (ig/ref :app.db/pool)
-       :executor (ig/ref :app.worker/executor)}
+   :app.loggers.sentry/reporter
+   {:dsn                (cf/get :sentry-dsn)
+    :trace-sample-rate  (cf/get :sentry-trace-sample-rate 1.0)
+    :attach-stack-trace (cf/get :sentry-attach-stack-trace false)
+    :debug              (cf/get :sentry-debug false)
+    :receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
 
-      :app.telemetry/server
-      {:port    (:telemetry-server-port config 6063)
-       :handler (ig/ref :app.telemetry/handler)
-       :name    "telemetry"}})))
+   :app.storage/storage
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)
 
-(defmethod ig/init-key :default [_ data] data)
-(defmethod ig/prep-key :default
-  [_ data]
-  (if (map? data)
-    (d/without-nils data)
-    data))
+    :backends {
+               :assets-s3 (ig/ref [::assets :app.storage.s3/backend])
+               :assets-db (ig/ref [::assets :app.storage.db/backend])
+               :assets-fs (ig/ref [::assets :app.storage.fs/backend])
+               :tmp       (ig/ref [::tmp  :app.storage.fs/backend])
+               :fdata-s3  (ig/ref [::fdata :app.storage.s3/backend])
+
+               ;; keep this for backward compatibility
+               :s3        (ig/ref [::assets :app.storage.s3/backend])
+               :fs        (ig/ref [::assets :app.storage.fs/backend])}}
+
+   [::fdata :app.storage.s3/backend]
+   {:region (cf/get :storage-fdata-s3-region)
+    :bucket (cf/get :storage-fdata-s3-bucket)
+    :prefix (cf/get :storage-fdata-s3-prefix)}
+
+   [::assets :app.storage.s3/backend]
+   {:region (cf/get :storage-assets-s3-region)
+    :bucket (cf/get :storage-assets-s3-bucket)}
+
+   [::assets :app.storage.fs/backend]
+   {:directory (cf/get :storage-assets-fs-directory)}
+
+   [::tmp :app.storage.fs/backend]
+   {:directory "/tmp/penpot"}
+
+   [::assets :app.storage.db/backend]
+   {:pool (ig/ref :app.db/pool)}})
 
 (def system nil)
 
 (defn start
   []
-  (let [system-config (build-system-config cfg/config)]
-    (ig/load-namespaces system-config)
-    (alter-var-root #'system (fn [sys]
-                               (when sys (ig/halt! sys))
-                               (-> system-config
-                                   (ig/prep)
-                                   (ig/init))))
-    (log/infof "Welcome to penpot! Version: '%s'."
-               (:full cfg/version))))
+  (ig/load-namespaces system-config)
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             (-> system-config
+                                 (ig/prep)
+                                 (ig/init))))
+  (l/info :msg "welcome to penpot"
+          :version (:full cf/version)))
 
 (defn stop
   []
@@ -342,14 +353,6 @@
                              (when sys (ig/halt! sys))
                              nil)))
 
-(prefer-method print-method
-               clojure.lang.IRecord
-               clojure.lang.IDeref)
-
-(prefer-method pprint/simple-dispatch
-               clojure.lang.IPersistentMap
-               clojure.lang.IDeref)
-
 (defn -main
   [& _args]
   (start))

backend/src/app/media.clj

@@ -2,34 +2,38 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.media
-  "Media postprocessing."
+  "Media & Font postprocessing."
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.media :as cm]
    [app.common.spec :as us]
-   [app.rlimits :as rlm]
-   [app.svgparse :as svg]
+   [app.config :as cf]
+   [app.util.svg :as svg]
+   [buddy.core.bytes :as bb]
+   [buddy.core.codecs :as bc]
+   [clojure.java.io :as io]
+   [clojure.java.shell :as sh]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.core :as fs])
   (:import
    java.io.ByteArrayInputStream
+   java.io.OutputStream
+   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-;; --- Generic specs
+(s/def ::image-content-type cm/valid-image-types)
+(s/def ::font-content-type cm/valid-font-types)
 
 (s/def :internal.http.upload/filename ::us/string)
 (s/def :internal.http.upload/size ::us/integer)
-(s/def :internal.http.upload/content-type cm/valid-media-types)
+(s/def :internal.http.upload/content-type ::us/string)
 (s/def :internal.http.upload/tempfile any?)
 
 (s/def ::upload
@@ -38,8 +42,37 @@
                    :internal.http.upload/tempfile
                    :internal.http.upload/content-type]))
 
+(defn validate-media-type
+  ([mtype] (validate-media-type mtype cm/valid-image-types))
+  ([mtype allowed]
+   (when-not (contains? allowed mtype)
+     (ex/raise :type :validation
+               :code :media-type-not-allowed
+               :hint "Seems like you are uploading an invalid media object"))))
 
+(defmulti process :cmd)
+(defmulti process-error class)
+
+(defmethod process :default
+  [{:keys [cmd] :as params}]
+  (ex/raise :type :internal
+            :code :not-implemented
+            :hint (str/fmt "No impl found for process cmd: %s" cmd)))
+
+(defmethod process-error :default
+  [error]
+  (throw error))
+
+(defn run
+  [params]
+  (try
+    (process params)
+    (catch Throwable e
+      (process-error e))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; --- Thumbnails Generation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::cmd keyword?)
 
@@ -77,11 +110,9 @@
       (assoc params
              :format format
              :mtype  (cm/format->mtype format)
-             :size   (alength thumbnail-data)
+             :size   (alength ^bytes thumbnail-data)
              :data   (ByteArrayInputStream. thumbnail-data)))))
 
-(defmulti process :cmd)
-
 (defmethod process :generic-thumbnail
   [{:keys [quality width height] :as params}]
   (us/assert ::thumbnail-params params)
@@ -89,7 +120,7 @@
              (.addImage)
              (.autoOrient)
              (.strip)
-             (.thumbnail (int width) (int height) ">")
+             (.thumbnail ^Integer (int width) ^Integer (int height) ">")
              (.quality (double quality))
              (.addImage))]
     (generic-process (assoc params :operation op))))
@@ -101,7 +132,7 @@
              (.addImage)
              (.autoOrient)
              (.strip)
-             (.thumbnail (int width) (int height) "^")
+             (.thumbnail ^Integer (int width) ^Integer (int height) "^")
              (.gravity "center")
              (.extent (int width) (int height))
              (.quality (double quality))
@@ -141,11 +172,10 @@
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [data (svg/parse (slurp path))
-            info (get-basic-info-from-svg data)]
+      (let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
-                    :code :unable-to-retrieve-dimensions
+                    :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
         (assoc info :mtype mtype))
 
@@ -157,31 +187,147 @@
                     :code :media-type-mismatch
                     :hint (str "Seems like you are uploading a file whose content does not match the extension."
                                "Expected: " mtype ". Got: " mtype')))
-        {:width  (.getImageWidth instance)
-         :height (.getImageHeight instance)
+        ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
+        ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
+        ;; any frame.
+        {:width  (.getPageWidth instance)
+         :height (.getPageHeight instance)
          :mtype  mtype}))))
 
-(defmethod process :default
-  [{:keys [cmd] :as params}]
-  (ex/raise :type :internal
-            :code :not-implemented
-            :hint (str "No impl found for process cmd:" cmd)))
+(defmethod process-error org.im4java.core.InfoException
+  [error]
+  (ex/raise :type :validation
+            :code :invalid-image
+            :hint "invalid image"
+            :cause error))
 
-(defn run
-  [{:keys [rlimits]} params]
-  (us/assert map? rlimits)
-  (let [rlimit (get rlimits :image)]
-    (when-not rlimit
-      (ex/raise :type :internal
-                :code :rlimit-not-configured
-                :hint ":image rlimit not configured"))
-    (rlm/execute rlimit (process params))))
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Fonts Generation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; --- Utility functions
+(defmethod process :generate-fonts
+  [{:keys [input] :as params}]
+  (letfn [(ttf->otf [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot")
+                  output-file (fs/path (str input-file ".otf"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "fontforge" "-lang=ff" "-c"
+                                     (str/fmt "Open('%s'); Generate('%s')"
+                                              (str input-file)
+                                              (str output-file)))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
+
+
+          (otf->ttf [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot")
+                  output-file (fs/path (str input-file ".ttf"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "fontforge" "-lang=ff" "-c"
+                                     (str/fmt "Open('%s'); Generate('%s')"
+                                              (str input-file)
+                                              (str output-file)))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
+
+          (ttf-or-otf->woff [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  output-file (fs/path (str input-file ".woff"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "sfnt2woff" (str input-file))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
+
+          (ttf-or-otf->woff2 [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  output-file (fs/path (str input-file ".woff2"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "woff2_compress" (str input-file))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
+
+          (woff->sfnt [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "woff2sfnt" (str input-file)
+                                     :out-enc :bytes)]
+              (when (zero? (:exit res))
+                (:out res))))
+
+          ;; Documented here:
+          ;; https://docs.microsoft.com/en-us/typography/opentype/spec/otff#table-directory
+          (get-sfnt-type [data]
+            (let [buff (bb/slice data 0 4)
+                  type (bc/bytes->hex buff)]
+              (case type
+                "4f54544f" :otf
+                "00010000" :ttf
+                (ex/raise :type :internal
+                          :code :unexpected-data
+                          :hint "unexpected font data"))))
+
+          (gen-if-nil [val factory]
+            (if (nil? val)
+              (factory)
+              val))]
+
+    (let [current (into #{} (keys input))]
+      (cond
+        (contains? current "font/ttf")
+        (let [data (get input "font/ttf")]
+          (-> input
+              (update "font/otf" gen-if-nil #(ttf->otf data))
+              (update "font/woff" gen-if-nil #(ttf-or-otf->woff data))
+              (assoc "font/woff2" (ttf-or-otf->woff2 data))))
+
+        (contains? current "font/otf")
+        (let [data (get input "font/otf")]
+          (-> input
+              (update "font/woff" gen-if-nil #(ttf-or-otf->woff data))
+              (assoc "font/ttf" (otf->ttf data))
+              (assoc "font/woff2" (ttf-or-otf->woff2 data))))
+
+        (contains? current "font/woff")
+        (let [data (get input "font/woff")
+              sfnt (woff->sfnt data)]
+          (when-not sfnt
+            (ex/raise :type :validation
+                      :code :invalid-woff-file
+                      :hint "invalid woff file"))
+          (let [stype (get-sfnt-type sfnt)]
+            (cond-> input
+              true
+              (-> (assoc "font/woff" data)
+                  (assoc "font/woff2" (ttf-or-otf->woff2 sfnt)))
+
+              (= stype :otf)
+              (-> (assoc "font/otf" sfnt)
+                  (assoc "font/ttf" (otf->ttf sfnt)))
+
+              (= stype :ttf)
+              (-> (assoc "font/otf" (ttf->otf sfnt))
+                  (assoc "font/ttf" sfnt)))))))))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Utility functions
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn configure-assets-storage
+  "Given storage map, returns a storage configured with the appropriate
+  backend for assets."
+  [storage conn]
+  (-> storage
+      (assoc :conn conn)
+      (assoc :backend (cf/get :assets-storage-backend :assets-fs))))
 
-(defn validate-media-type
-  [media-type]
-  (when-not (cm/valid-media-types media-type)
-    (ex/raise :type :validation
-              :code :media-type-not-allowed
-              :hint "Seems like you are uploading an invalid media object")))

backend/src/app/metrics.clj

@@ -2,25 +2,20 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020-2021 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.metrics
   (:require
    [app.common.exceptions :as ex]
-   [app.util.time :as dt]
-   [app.worker]
+   [app.common.logging :as l]
    [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [integrant.core :as ig]
-   [next.jdbc :as jdbc])
+   [integrant.core :as ig])
   (:import
    io.prometheus.client.CollectorRegistry
    io.prometheus.client.Counter
    io.prometheus.client.Gauge
    io.prometheus.client.Summary
+   io.prometheus.client.Histogram
    io.prometheus.client.exporter.common.TextFormat
    io.prometheus.client.hotspot.DefaultExports
    io.prometheus.client.jetty.JettyStatisticsCollector
@@ -30,40 +25,74 @@
 (declare instrument-vars!)
 (declare instrument)
 (declare create-registry)
+(declare create)
+(declare handler)
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Defaults
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+(def default-metrics
+  {:profile-register
+   {:name "actions_profile_register_count"
+    :help "A global counter of user registrations."
+    :type :counter}
+
+   :profile-activation
+   {:name "actions_profile_activation_count"
+    :help "A global counter of profile activations"
+    :type :counter}
+
+   :update-file-changes
+   {:name "rpc_update_file_changes_total"
+    :help "A total number of changes submitted to update-file."
+    :type :counter}
+
+   :update-file-bytes-processed
+   {:name "rpc_update_file_bytes_processed_total"
+    :help "A total number of bytes processed by update-file."
+    :type :counter}
+
+   :websocket-active-connections
+   {:name "websocket_active_connections"
+    :help "Active websocket connections gauge"
+    :type :gauge}
+
+   :websocket-messages-total
+   {:name "websocket_message_total"
+    :help "Counter of processed messages."
+    :labels ["op"]
+    :type :counter}
+
+   :websocket-session-timing
+   {:name "websocket_session_timing"
+    :help "Websocket session timing (seconds)."
+    :quantiles []
+    :type :summary}})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Entry Point
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- instrument-jdbc!
-  [registry]
-  (instrument-vars!
-   [#'next.jdbc/execute-one!
-    #'next.jdbc/execute!]
-   {:registry registry
-    :type :counter
-    :name "database_query_counter"
-    :help "An absolute counter of database queries."}))
+(defmethod ig/init-key ::metrics
+  [_ _]
+  (l/info :action "initialize metrics")
+  (let [registry    (create-registry)
+        definitions (reduce-kv (fn [res k v]
+                                 (->> (assoc v :registry registry)
+                                      (create)
+                                      (assoc res k)))
+                               {}
+                               default-metrics)]
+    {:handler (partial handler registry)
+     :definitions definitions
+     :registry registry}))
 
-(defn- instrument-workers!
-  [registry]
-  (instrument-vars!
-   [#'app.worker/run-task]
-   {:registry registry
-    :type :summary
-    :name "worker_task_checkout_millis"
-    :help "Latency measured between scheduld_at and execution time."
-    :wrap (fn [rootf mobj]
-            (let [mdata (meta rootf)
-                  origf (::original mdata rootf)]
-              (with-meta
-                (fn [tasks item]
-                  (let [now (inst-ms (dt/now))
-                        sat (inst-ms (:scheduled-at item))]
-                    (mobj :observe (- now sat))
-                    (origf tasks item)))
-                {::original origf})))}))
+(s/def ::handler fn?)
+(s/def ::registry #(instance? CollectorRegistry %))
+(s/def ::metrics
+  (s/keys :req-un [::registry ::handler]))
 
 (defn- handler
   [registry _request]
@@ -73,21 +102,6 @@
     {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
      :body (.toString writer)}))
 
-(defmethod ig/init-key ::metrics
-  [_ _cfg]
-  (log/infof "Initializing prometheus registry and instrumentation.")
-  (let [registry (create-registry)]
-    (instrument-workers! registry)
-    (instrument-jdbc! registry)
-    {:handler (partial handler registry)
-     :registry registry}))
-
-(s/def ::handler fn?)
-(s/def ::registry #(instance? CollectorRegistry %))
-(s/def ::metrics
-  (s/keys :req-un [::registry ::handler]))
-
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Implementation
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -116,18 +130,14 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (.inc ^Counter instance))
-
-      (invoke [_ cmd labels]
-        (.. ^Counter instance
-            (labels labels)
-            (inc))))))
+    {::instance instance
+     ::fn (fn [{:keys [by labels] :or {by 1}}]
+            (if labels
+              (.. ^Counter instance
+                  (labels (into-array String labels))
+                  (inc by))
+              (.inc ^Counter instance by)))}))
 
 (defn make-gauge
   [{:keys [name help registry reg labels] :as props}]
@@ -139,51 +149,75 @@
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
 
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
+    {::instance instance
+     ::fn (fn [{:keys [cmd by labels] :or {by 1}}]
+            (if labels
+              (let [labels (into-array String [labels])]
+                (case cmd
+                  :inc (.. ^Gauge instance (labels labels) (inc by))
+                  :dec (.. ^Gauge instance (labels labels) (dec by))))
+              (case cmd
+                :inc (.inc ^Gauge instance by)
+                :dec (.dec ^Gauge instance by))))}))
 
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (case cmd
-          :inc (.inc ^Gauge instance)
-          :dec (.dec ^Gauge instance)))
-
-      (invoke [_ cmd labels]
-        (case cmd
-          :inc (.. ^Gauge instance (labels labels) (inc))
-          :dec (.. ^Gauge instance (labels labels) (dec)))))))
+(def default-quantiles
+  [[0.75 0.02]
+   [0.99 0.001]])
 
 (defn make-summary
-  [{:keys [name help registry reg labels] :as props}]
+  [{:keys [name help registry reg labels max-age quantiles buckets]
+    :or {max-age 3600 buckets 6 quantiles default-quantiles} :as props}]
   (let [registry (or registry reg)
         instance (doto (Summary/build)
                    (.name name)
-                   (.help help)
-                   (.quantile 0.75 0.02)
-                   (.quantile 0.99 0.001))
+                   (.help help))
+        _        (when (seq quantiles)
+                   (.maxAgeSeconds ^Summary instance max-age)
+                   (.ageBuckets ^Summary instance buckets))
+        _        (doseq [[q e] quantiles]
+                   (.quantile ^Summary instance q e))
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd val]
-        (.observe ^Summary instance val))
+    {::instance instance
+     ::fn (fn [{:keys [val labels]}]
+            (if labels
+              (.. ^Summary instance
+                  (labels (into-array String labels))
+                  (observe val))
+              (.observe ^Summary instance val)))}))
 
-      (invoke [_ cmd val labels]
-        (.. ^Summary instance
-            (labels labels)
-            (observe val))))))
+(def default-histogram-buckets
+  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+
+(defn make-histogram
+  [{:keys [name help registry reg labels buckets]
+    :or {buckets default-histogram-buckets}}]
+  (let [registry (or registry reg)
+        instance (doto (Histogram/build)
+                   (.name name)
+                   (.help help)
+                   (.buckets (into-array Double/TYPE buckets)))
+        _        (when (seq labels)
+                   (.labelNames instance (into-array String labels)))
+        instance (.register instance registry)]
+
+    {::instance instance
+     ::fn (fn [{:keys [val labels]}]
+            (if labels
+              (.. ^Histogram instance
+                  (labels (into-array String labels))
+                  (observe val))
+              (.observe ^Histogram instance val)))}))
 
 (defn create
   [{:keys [type] :as props}]
   (case type
     :counter (make-counter props)
     :gauge   (make-gauge props)
-    :summary (make-summary props)))
+    :summary (make-summary props)
+    :histogram (make-histogram props)))
 
 (defn wrap-counter
   ([rootf mobj]
@@ -192,29 +226,34 @@
      (with-meta
        (fn
          ([a]
-          (mobj :inc)
+          ((::fn mobj) nil)
           (origf a))
          ([a b]
-          (mobj :inc)
+          ((::fn mobj) nil)
           (origf a b))
-         ([a b & more]
-          (mobj :inc)
-          (apply origf a b more)))
+         ([a b c]
+          ((::fn mobj) nil)
+          (origf a b c))
+         ([a b c d]
+          ((::fn mobj) nil)
+          (origf a b c d))
+         ([a b c d & more]
+          ((::fn mobj) nil)
+          (apply origf a b c d more)))
        (assoc mdata ::original origf))))
   ([rootf mobj labels]
    (let [mdata  (meta rootf)
-         labels (into-array String labels)
          origf  (::original mdata rootf)]
      (with-meta
        (fn
          ([a]
-          (mobj :inc labels)
+          ((::fn mobj) {:labels labels})
           (origf a))
          ([a b]
-          (mobj :inc labels)
+          ((::fn mobj) {:labels labels})
           (origf a b))
          ([a b & more]
-          (mobj :inc labels)
+          ((::fn mobj) {:labels labels})
           (apply origf a b more)))
        (assoc mdata ::original origf)))))
 
@@ -227,46 +266,45 @@
          ([a]
          (with-measure
            :expr (origf a)
-           :cb   #(mobj :observe %)))
+           :cb   #((::fn mobj) {:val %})))
          ([a b]
           (with-measure
             :expr (origf a b)
-            :cb   #(mobj :observe %)))
+            :cb   #((::fn mobj) {:val %})))
          ([a b & more]
           (with-measure
             :expr (apply origf a b more)
-            :cb   #(mobj :observe %))))
+            :cb   #((::fn mobj) {:val %}))))
        (assoc mdata ::original origf))))
 
   ([rootf mobj labels]
    (let [mdata  (meta rootf)
-         labels (into-array String labels)
          origf  (::original mdata rootf)]
      (with-meta
        (fn
          ([a]
          (with-measure
            :expr (origf a)
-           :cb   #(mobj :observe % labels)))
+           :cb   #((::fn mobj) {:val % :labels labels})))
          ([a b]
           (with-measure
             :expr (origf a b)
-            :cb   #(mobj :observe % labels)))
+            :cb   #((::fn mobj) {:val % :labels labels})))
          ([a b & more]
           (with-measure
             :expr (apply origf a b more)
-            :cb   #(mobj :observe % labels))))
+            :cb   #((::fn mobj) {:val % :labels labels}))))
        (assoc mdata ::original origf)))))
 
 (defn instrument-vars!
   [vars {:keys [wrap] :as props}]
   (let [obj (create props)]
     (cond
-      (instance? Counter @obj)
+      (instance? Counter (::instance obj))
       (doseq [var vars]
         (alter-var-root var (or wrap wrap-counter) obj))
 
-      (instance? Summary @obj)
+      (instance? Summary (::instance obj))
       (doseq [var vars]
         (alter-var-root var (or wrap wrap-summary) obj))
 
@@ -277,10 +315,13 @@
   [f {:keys [wrap] :as props}]
   (let [obj (create props)]
     (cond
-      (instance? Counter @obj)
+      (instance? Counter (::instance obj))
       ((or wrap wrap-counter) f obj)
 
-      (instance? Summary @obj)
+      (instance? Summary (::instance obj))
+      ((or wrap wrap-summary) f obj)
+
+      (instance? Histogram (::instance obj))
       ((or wrap wrap-summary) f obj)
 
       :else

backend/src/app/migrations.clj

@@ -2,10 +2,7 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.migrations
   (:require
@@ -145,6 +142,69 @@
 
    {:name "0044-add-storage-refcount"
     :fn (mg/resource "app/migrations/sql/0044-add-storage-refcount.sql")}
+
+   {:name "0045-add-index-to-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0045-add-index-to-file-change-table.sql")}
+
+   {:name "0046-add-profile-complaint-table"
+    :fn (mg/resource "app/migrations/sql/0046-add-profile-complaint-table.sql")}
+
+   {:name "0047-mod-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0047-mod-file-change-table.sql")}
+
+   {:name "0048-mod-storage-tables"
+    :fn (mg/resource "app/migrations/sql/0048-mod-storage-tables.sql")}
+
+   {:name "0049-mod-http-session-table"
+    :fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")}
+
+   {:name "0050-mod-server-prop-table"
+    :fn (mg/resource "app/migrations/sql/0050-mod-server-prop-table.sql")}
+
+   {:name "0051-mod-file-library-rel-table"
+    :fn (mg/resource "app/migrations/sql/0051-mod-file-library-rel-table.sql")}
+
+   {:name "0052-del-legacy-user-and-team"
+    :fn (mg/resource "app/migrations/sql/0052-del-legacy-user-and-team.sql")}
+
+   {:name "0053-add-team-font-variant-table"
+    :fn (mg/resource "app/migrations/sql/0053-add-team-font-variant-table.sql")}
+
+   {:name "0054-add-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0054-add-audit-log-table.sql")}
+
+   {:name "0055-mod-file-media-object-table"
+    :fn (mg/resource "app/migrations/sql/0055-mod-file-media-object-table.sql")}
+
+   {:name "0056-add-missing-index-on-deleted-at"
+    :fn (mg/resource "app/migrations/sql/0056-add-missing-index-on-deleted-at.sql")}
+
+   {:name "0057-del-profile-on-delete-trigger"
+    :fn (mg/resource "app/migrations/sql/0057-del-profile-on-delete-trigger.sql")}
+
+   {:name "0058-del-team-on-delete-trigger"
+    :fn (mg/resource "app/migrations/sql/0058-del-team-on-delete-trigger.sql")}
+
+   {:name "0059-mod-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0059-mod-audit-log-table.sql")}
+
+   {:name "0060-mod-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0060-mod-file-change-table.sql")}
+
+   {:name "0061-mod-file-table"
+    :fn (mg/resource "app/migrations/sql/0061-mod-file-table.sql")}
+
+   {:name "0062-fix-metadata-media"
+    :fn (mg/resource "app/migrations/sql/0062-fix-metadata-media.sql")}
+
+   {:name "0063-add-share-link-table"
+    :fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")}
+
+   {:name "0064-mod-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")}
+
+   {:name "0065-add-trivial-spelling-fixes"
+    :fn (mg/resource "app/migrations/sql/0065-add-trivial-spelling-fixes.sql")}
    ])
 
 

backend/src/app/migrations/migration_0023.clj

@@ -2,10 +2,7 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.migrations.migration-0023
   (:require

backend/src/app/migrations/sql/0015-improve-tasks-tables.sql

@@ -1,4 +1,4 @@
-DROP TABLE task;
+DROP TABLE IF EXISTS task;
 
 CREATE TABLE task (
   id uuid DEFAULT uuid_generate_v4(),
@@ -27,3 +27,11 @@ CREATE TABLE task_default partition OF task default;
 CREATE INDEX task__scheduled_at__queue__idx
     ON task (scheduled_at, queue)
  WHERE status = 'new' or status = 'retry';
+
+ALTER TABLE task
+  ALTER COLUMN queue SET STORAGE external,
+  ALTER COLUMN name SET STORAGE external,
+  ALTER COLUMN props SET STORAGE external,
+  ALTER COLUMN status SET STORAGE external,
+  ALTER COLUMN error SET STORAGE external;
+

backend/src/app/migrations/sql/0019-add-improved-scheduled-tasks.sql

@@ -1,4 +1,4 @@
-DROP TABLE scheduled_task;
+DROP TABLE IF EXISTS scheduled_task;
 
 CREATE TABLE scheduled_task (
   id text PRIMARY KEY,
@@ -22,3 +22,7 @@ CREATE TABLE scheduled_task_history (
 
 CREATE INDEX scheduled_task_history__task_id__idx
     ON scheduled_task_history(task_id);
+
+ALTER TABLE scheduled_task
+  ALTER COLUMN id SET STORAGE external,
+  ALTER COLUMN cron_expr SET STORAGE external;

backend/src/app/migrations/sql/0035-add-storage-tables.sql

@@ -10,13 +10,19 @@ CREATE TABLE storage_object (
   metadata jsonb NULL DEFAULT NULL
 );
 
+CREATE INDEX storage_object__id__deleted_at__idx
+    ON storage_object(id, deleted_at)
+ WHERE deleted_at IS NOT null;
+
 CREATE TABLE storage_data (
   id uuid PRIMARY KEY REFERENCES storage_object (id) ON DELETE CASCADE,
   data bytea NOT NULL
 );
 
+CREATE INDEX storage_data__id__idx ON storage_data(id);
+
 -- Table used for store inflight upload ids, for later recheck and
--- delete possible staled files that exists on the phisical storage
+-- delete possible staled files that exists on the physical storage
 -- but does not exists in the 'storage_object' table.
 
 CREATE TABLE storage_pending (
@@ -28,8 +34,3 @@ CREATE TABLE storage_pending (
   PRIMARY KEY (created_at, id)
 );
 
-CREATE INDEX storage_data__id__idx ON storage_data(id);
-CREATE INDEX storage_object__id__deleted_at__idx
-    ON storage_object(id, deleted_at)
- WHERE deleted_at IS NOT null;
-

backend/src/app/migrations/sql/0041-mod-pg-storage-options.sql

@@ -27,17 +27,6 @@ ALTER TABLE comment_thread
   ALTER COLUMN participants SET STORAGE external,
   ALTER COLUMN page_name SET STORAGE external;
 
-ALTER TABLE task
-  ALTER COLUMN queue SET STORAGE external,
-  ALTER COLUMN name SET STORAGE external,
-  ALTER COLUMN props SET STORAGE external,
-  ALTER COLUMN status SET STORAGE external,
-  ALTER COLUMN error SET STORAGE external;
-
-ALTER TABLE scheduled_task
-  ALTER COLUMN id SET STORAGE external,
-  ALTER COLUMN cron_expr SET STORAGE external;
-
 ALTER TABLE http_session
   ALTER COLUMN id SET STORAGE external,
   ALTER COLUMN user_agent SET STORAGE external;

backend/src/app/migrations/sql/0044-add-storage-refcount.sql

@@ -5,9 +5,6 @@ CREATE INDEX storage_object__id_touched_at__idx
     ON storage_object (touched_at, id)
  WHERE touched_at IS NOT NULL;
 
--- DROP TRIGGER file_media_object__on_delete__tgr ON file_media_object CASCADE;
--- DROP FUNCTION on_delete_file_media_object () ;
-
 CREATE OR REPLACE FUNCTION on_delete_file_media_object()
   RETURNS TRIGGER AS $func$
   BEGIN

backend/src/app/migrations/sql/0045-add-index-to-file-change-table.sql

@@ -0,0 +1,2 @@
+CREATE INDEX file_change__created_at_idx
+    ON file_change (created_at);

backend/src/app/migrations/sql/0046-add-profile-complaint-table.sql

@@ -0,0 +1,45 @@
+CREATE TABLE profile_complaint_report (
+  profile_id uuid NOT NULL REFERENCES profile(id) ON DELETE CASCADE,
+  created_at timestamptz NOT NULL DEFAULT now(),
+
+  type text NOT NULL,
+  content jsonb,
+
+  PRIMARY KEY (profile_id, created_at)
+);
+
+ALTER TABLE profile_complaint_report
+  ALTER COLUMN type SET STORAGE external,
+  ALTER COLUMN content SET STORAGE external;
+
+ALTER TABLE profile
+  ADD COLUMN is_muted boolean DEFAULT false,
+  ADD COLUMN auth_backend text NULL;
+
+ALTER TABLE profile
+  ALTER COLUMN auth_backend SET STORAGE external;
+
+UPDATE profile
+   SET auth_backend = 'google'
+ WHERE password = '!';
+
+UPDATE profile
+   SET auth_backend = 'penpot'
+ WHERE password != '!';
+
+-- Table storing a permanent complaint table for register all
+-- permanent bounces and spam reports (complaints) and avoid sending
+-- more emails there.
+CREATE TABLE global_complaint_report (
+  email text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+
+  type text NOT NULL,
+  content jsonb,
+
+  PRIMARY KEY (email, created_at)
+);
+
+ALTER TABLE global_complaint_report
+  ALTER COLUMN type SET STORAGE external,
+  ALTER COLUMN content SET STORAGE external;

backend/src/app/migrations/sql/0047-mod-file-change-table.sql

@@ -0,0 +1,16 @@
+--- Helps on the lagged changes query on update-file rpc
+CREATE INDEX file_change__file_id__revn__idx ON file_change (file_id, revn);
+
+--- Drop redundant index
+DROP INDEX page_change_file_id_idx;
+
+--- Add profile_id field.
+ALTER TABLE file_change
+  ADD COLUMN profile_id uuid NULL REFERENCES profile (id) ON DELETE SET NULL;
+
+CREATE INDEX file_change__profile_id__idx
+    ON file_change (profile_id)
+ WHERE profile_id IS NOT NULL;
+
+--- Fix naming
+ALTER INDEX file_change__created_at_idx RENAME TO file_change__created_at__idx;

backend/src/app/migrations/sql/0048-mod-storage-tables.sql

@@ -0,0 +1,9 @@
+--- Drop redundant index already covered by primary key
+DROP INDEX storage_data__id__idx;
+
+--- Replace not efficient index with more efficient one
+DROP INDEX storage_object__id__deleted_at__idx;
+
+CREATE INDEX storage_object__id__deleted_at__idx
+    ON storage_object(deleted_at, id)
+ WHERE deleted_at IS NOT NULL;

backend/src/app/migrations/sql/0049-mod-http-session-table.sql

@@ -0,0 +1,6 @@
+ALTER TABLE http_session
+  ADD COLUMN updated_at timestamptz NULL;
+
+CREATE INDEX http_session__updated_at__idx
+    ON http_session (updated_at)
+ WHERE updated_at IS NOT NULL;

backend/src/app/migrations/sql/0050-mod-server-prop-table.sql

@@ -0,0 +1,4 @@
+ALTER TABLE server_prop
+  ADD COLUMN preload boolean DEFAULT false;
+
+UPDATE server_prop SET preload = true;