Merge pull request #2209 from penpot/palba-protect-profile-url

🐛 Fix opening profile URL while signed out takes to "your account" section

.circleci/config.yml

@@ -2,22 +2,16 @@ version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
       - image: penpotapp/devenv:latest
-
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
-      - image: circleci/postgres:13.3-ram
+      - image: cimg/postgres:13.5
         environment:
           POSTGRES_USER: penpot_test
           POSTGRES_PASSWORD: penpot_test
           POSTGRES_DB: penpot_test
-
-      - image: circleci/redis:6.0.8
+      - image: cimg/redis:6.2.6
 
     working_directory: ~/repo
+    resource_class: large
 
     environment:
       # Customize the JVM maximum heap limit
@@ -33,20 +27,35 @@ jobs:
            # fallback to using the latest cache if no exact match is found
            - v1-dependencies-
 
+      - run: cd .clj-kondo && cat config.edn
+
       - run:
           name: common lint
           working_directory: "./common"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       - run:
           name: frontend lint
           working_directory: "./frontend"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
+
+      - run:
+          name: frontend styles prettier
+          working_directory: "./frontend"
+          command: |
+            yarn install
+            yarn run lint-scss
 
       - run:
           name: backend lint
           working_directory: "./backend"
-          command: "clj-kondo --parallel --lint src/"
+          command: |
+            clj-kondo --version
+            clj-kondo --parallel --lint src/
 
       # run backend test
       - run:
@@ -68,21 +77,27 @@ jobs:
             node target/tests.js
 
           environment:
-            JAVA_HOME: /usr/lib/jvm/openjdk16
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
+
+      # - run:
+      #     working_directory: "./common"
+      #     name: common tests (cljs)
+      #     command: |
+      #       yarn install
+      #       yarn run compile-test
+      #       node target/test.js
+      # 
+      #     environment:
+      #       PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
 
       - run:
           working_directory: "./common"
-          name: common tests
+          name: common tests (clj)
           command: |
-            yarn install
-            clojure -M:dev:shadow-cljs compile test
-            node target/tests.js
             clojure -X:dev:test
 
           environment:
-            JAVA_HOME: /usr/lib/jvm/openjdk16
-            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
 
       - save_cache:
          paths:

.clj-kondo/config.edn

@@ -1,15 +1,19 @@
 {:lint-as
  {promesa.core/let clojure.core/let
+  promesa.core/->> clojure.core/->>
+  promesa.core/-> clojure.core/->
   rumext.alpha/defc clojure.core/defn
   rumext.alpha/fnc clojure.core/fn
   app.common.data/export clojure.core/def
-  app.db/with-atomic clojure.core/with-open}
+  app.db/with-atomic clojure.core/with-open
+  app.common.data.macros/get-in clojure.core/get-in
+  app.common.data.macros/select-keys clojure.core/select-keys
+  app.common.logging/with-context clojure.core/do}
 
  :hooks
  {:analyze-call
-  {app.common.data/export hooks.export/export
+  {app.common.data.macros/export hooks.export/export
    potok.core/reify hooks.export/potok-reify
-   cljs.core/specify! hooks.export/clojure-specify
    app.util.services/defmethod hooks.export/service-defmethod
    }}
 
@@ -17,12 +21,16 @@
  {:exclude-files
   ["data_readers.clj"
    "app/util/perf.cljs"
+   "app/common/logging.cljc"
    "app/common/exceptions.cljc"]}
 
  :linters
  {:unsorted-required-namespaces
   {:level :warning}
 
+  :potok/reify-type
+  {:level :error}
+
   :unresolved-namespace
   {:level :warning
    :exclude [data_readers]}
@@ -30,6 +38,9 @@
   :single-key-in
   {:level :warning}
 
+  :non-arg-vec-return-type-hint
+  {:level :off}
+
   :redundant-do
   {:level :off}
 

.clj-kondo/hooks/export.clj

@@ -10,15 +10,34 @@
                   sname])]
     {:node result}))
 
+(def registry (atom {}))
+
 (defn potok-reify
-  [{:keys [:node]}]
+  [{:keys [:node :filename] :as params}]
   (let [[rnode rtype & other] (:children node)
-        result  (api/list-node
-                 (into [(api/token-node (symbol "deftype"))
-                        (api/token-node (gensym (name (:k rtype))))
-                        (api/vector-node [])]
-                       other))]
-    {:node result}))
+        rsym (symbol (str "event-type-" (name (:k rtype))))
+        reg  (get @registry filename #{})]
+    (when-not (:namespaced? rtype)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message "ptk/reify type should be namespaced"
+                           :type :potok/reify-type
+                           :row row
+                           :col col})))
+
+    (if (contains? reg rsym)
+      (let [{:keys [:row :col]} (meta rtype)]
+        (api/reg-finding! {:message (str "duplicate type: " (name (:k rtype)))
+                           :type :potok/reify-type
+                           :row row
+                           :col col}))
+      (swap! registry update filename (fnil conj #{}) rsym))
+
+    (let [result  (api/list-node
+                   (into [(api/token-node (symbol "deftype"))
+                          (api/token-node rsym)
+                          (api/vector-node [])]
+                         other))]
+      {:node result})))
 
 (defn clojure-specify
   [{:keys [:node]}]
@@ -32,18 +51,39 @@
 
 (defn service-defmethod
   [{:keys [:node]}]
-  (let [[rnode rtype & other] (:children node)
+  (let [[rnode rtype ?meta & other] (:children node)
         rsym    (gensym (name (:k rtype)))
-        result  (api/list-node
-                 [(api/token-node (symbol "do"))
-                  (api/list-node
-                   [(api/token-node (symbol "declare"))
-                    (api/token-node rsym)])
-                  (api/list-node
-                   (into [(api/token-node (symbol "defmethod"))
-                          (api/token-node rsym)
-                          rtype]
-                         other))])]
+
+        [?docs other] (if (api/string-node? ?meta)
+                        [?meta other]
+                        [nil (cons ?meta other)])
+
+        [?meta other] (let [?meta (first other)]
+                        (if (api/map-node? ?meta)
+                          [?meta (rest other)]
+                          [nil   other]))
+
+        nodes         [(api/token-node (symbol "do"))
+                       (api/list-node
+                        [(api/token-node (symbol "declare"))
+                         (api/token-node rsym)])
+
+                       (when ?docs
+                         (api/list-node
+                          [(api/token-node (symbol "comment")) ?docs]))
+
+                       (when ?meta
+                         (api/list-node
+                          [(api/token-node (symbol "reset-meta!"))
+                           (api/token-node rsym)
+                           ?meta]))
+                       (api/list-node
+                        (into [(api/token-node (symbol "defmethod"))
+                               (api/token-node rsym)
+                               rtype]
+                              other))]
+        result  (api/list-node (filterv some? nodes))]
+
+    ;; (prn "=====>" rtype)
+    ;; (prn (api/sexpr result))
     {:node result}))
-
-

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,49 +8,48 @@ assignees: ''
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
-
 **To Reproduce**
+
 Steps to reproduce the behavior:
 1.  Go to '...'
 2.  Click on '....'
 3.  Scroll down to '....'
-4.  See error
 
 **Expected behavior**
+
 A clear and concise description of what you expected to happen.
 
+**Actual behavior**
+
+A clear and concise description of what happens instead; what the bug is.
+ 
 **Screenshots**
+
 If applicable, add screenshots to help explain your problem.
 
 **Desktop (please complete the following information):**
-
--   OS: (e.g. iOS)
--   Browser (e.g. chrome, safari)
--   Version (e.g. 22)
+- OS (e.g. iOS):
+- Browser & version (e.g. Chrome 89.0):
 
 **Smartphone (please complete the following information):**
-
--   Device: (e.g. iPhone6)
--   OS: (e.g. iOS8.1)
--   Browser (e.g. stock browser, safari)
--   Version (e.g. 22)
+- Device & model (e.g. iPhone 6):
+- OS & version (e.g. iOS 8.1):
+- Browser & version (e.g. stock browser 22):
 
 **Environment (please complete the following information):**
-Specify if using SAAS (https://design.penpot.app) or self-hosted instance.
+- Host (e.g. https://design.penpot.app, local instance):
 
-If self-hosted instance, add OS and runtime information to help explain your problem.
+*If self-hosted:*
+- OS Version (e.g. Ubuntu 16.04):
+- Docker / Docker-compose version (e.g. Docker version 18.03.0-ce, build 0520e24):
+- Image version (e.g. Alpine):
 
--   OS Version: (e.g. Ubuntu 16.04)
+Docker commands or docker-compose file (if possible and if proceed.x):
+```
 
-Also provide Docker commands or docker-compose file if possible and if proceed.x
-
--   Docker / Docker-compose Version: (e.g. Docker version 18.03.0-ce, build 0520e24)
--   Image (e.g. alpine)
-
-**Frontend Stack Trace (if self-hosted)**
+```
 
+Frontend Stack Trace:
 <details>
 
 ```
@@ -59,8 +58,7 @@ Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 </details>
 
-**Backend Stack Trace (if self-hosted)**
-
+Backend Stack Trace:
 <details>
 
 ```
@@ -69,5 +67,6 @@ Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 </details>
 
-**Additional context**
-Add any other context about the problem here.
+**Additional context:**
+
+Any other context about the problem.

.gitignore

@@ -1,41 +1,55 @@
-figwheel_server.log
-*jar
 *-init.clj
+*.jar
+*.penpot
+*.orig
+.calva
+.clj-kondo
+.cpcache
 .lein-deps-sum
 .lein-failures
-.lein-repl-history
 .lein-plugins/
-.repl
+.lein-repl-history
+.lsp
 .nrepl-port
-.cpcache
+.nyc_output
 .rebel_readline_history
-/vendor/**/target
-/cd.md
-node_modules
-/backend/target/
-/backend/resources/public/media
-/backend/resources/public/assets
+.repl
+/.clj-kondo/.cache
+/_dump
+/backend/-
 /backend/assets/
 /backend/dist/
 /backend/logs/
-/backend/-
-/telemetry/
-/frontend/npm-debug.log
-/frontend/target/
-/frontend/dist/
-/frontend/out/
-/frontend/.shadow-cljs
-/frontend/resources/public/*
-/frontend/resources/fonts/experiments
-/exporter/target
-/exporter/.shadow-cljs
-/docker/images/bundle*
-/common/.shadow-cljs
-/common/target
-/.clj-kondo/.cache
+/backend/resources/public/assets
+/backend/resources/public/media
+/backend/target/
+/backend/builtin-templates
 /bundle*
-/media
+/cd.md
+/clj-profiler/
+/common/.shadow-cljs
+/common/coverage
+/common/target
 /deploy
+/docker/images/bundle*
+/exporter/.shadow-cljs
+/exporter/target
+/frontend/.shadow-cljs
+/frontend/package-lock.json
+/frontend/cypress/videos/*/
+/frontend/cypress/fixtures/validuser.json
+/frontend/dist/
+/frontend/npm-debug.log
+/frontend/out/
+/frontend/resources/fonts/experiments
+/frontend/resources/public/*
+/frontend/target/
+/frontend/cypress/videos/*/
+/media
+/telemetry/
+/vendor/**/target
+/vendor/svgclean/bundle*.js
 /web
-/_dump
-/vendor/svgclean/bundle*.js
+clj-profiler/
+figwheel_server.log
+node_modules

CONTRIBUTING.md

@@ -19,9 +19,9 @@ If you found a bug, please report it, as far as possible with:
 - a browser and the browser version used
 - a dev tools console exception stack trace (if it is available)
 
-If you found a bug that you consider better discuse in private (for
+If you found a bug that you consider better discuss in private (for
 example: security bugs), consider first send an email to
-`info@penpot.app`.
+`support@penpot.app`.
 
 **We don't have formal bug bounty program for security reports; this
 is an open source application and your contribution will be recognized
@@ -54,7 +54,7 @@ We will use the `easy fix` mark for tag for indicate issues that are
 easy for beginners.
 
 
-## Commit Message Guidelines ##
+## Commit Guidelines ##
 
 We have very precise rules over how our git commit messages can be formatted.
 
@@ -78,7 +78,6 @@ Where type is:
 - :ambulance: `:ambulance:` a commit that fixes critical bug
 - :books: `:books:` a commit that improves or adds documentation
 - :construction: `:construction:`: a wip commit
-- :construction_worker: `:construction_worker:` a commit with CI related stuff
 - :boom: `:boom:` a commit with breaking changes
 - :wrench: `:wrench:` a commit for config updates
 - :zap: `:zap:` a commit with performance improvements
@@ -91,12 +90,25 @@ More info:
  - https://gist.github.com/parmentf/035de27d6ed1dce0b36a
  - https://gist.github.com/rxaviers/7360908
 
-The subject should be:
+Each commit should have:
 
-- Use the imperative mood.
-- Capitalize the first letter.
-- Don't put a period at the end of the subject line.
-- Put a blank line between the subject line and the body.
+- A concise subject using imperative mood.
+- The subject should have capitalized the first letter, without period
+  at the end and no larger than 65 characters.
+- A blank line between the subject line and the body.
+- An entry on the CHANGES.md file if applicable, referencing the
+  github or taiga issue/user-story using the these same rules.
+
+Examples of good commit messags:
+
+- :bug: Fix unexpected error on launching modal
+- :bug: Set proper error message on generic error
+- :sparkles: Enable new modal for profile
+- :zap: Improve performance of dashboard navigation
+- :wrench: Update default backend configuration
+- :books: Add more documentation for authentication process
+- :ambulance: Fix critical bug on user registration process
+- :tada: Add new approach for user registration
 
 
 ## Code of conduct ##

README.md

@@ -70,9 +70,9 @@ You can ask and answer questions, have open-ended conversations, and follow alon
 
 ✉️ [Mail us](mailto:info@penpot.app)
 
-💬 [Github discussions](https://github.com/penpot/penpot/discussions)
+💬 [GitHub discussions](https://github.com/penpot/penpot/discussions)
 
-🐞 [Github issues](mailto:info@penpot.apphttps://github.com/penpot/penpot/issues)
+🐞 [GitHub issues](https://github.com/penpot/penpot/issues)
 
 ✍️️ [Gitter](https://gitter.im/penpot/community)
 
@@ -81,7 +81,7 @@ You can ask and answer questions, have open-ended conversations, and follow alon
 You can ask and answer questions, have open-ended conversations, and follow along on decisions affecting the project.
 Would you like to know more about Penpot? We recommend you to visit our youtube channel and learn more about the functionalities and possibilities of Penpot with our video tutorials.
 
-🎞️ [Youtube channel](https://www.youtube.com/channel/UCAqS8G72uv9P5HG1IfgnQ9g)
+🎞️ [YouTube channel](https://www.youtube.com/channel/UCAqS8G72uv9P5HG1IfgnQ9g)
 
 ## License ##
 

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please report security issues to `support@penpot.app`

backend/build.clj

@@ -0,0 +1,36 @@
+(ns build
+  (:refer-clojure :exclude [compile])
+  (:require
+   [clojure.tools.build.api :as b]
+   [clojure.java.io]))
+
+(def class-dir "target/classes")
+(def basis (b/create-basis {:project "deps.edn"}))
+(def jar-file "target/penpot.jar")
+
+(defn clean [_]
+  (b/delete {:path "target"}))
+
+(defn jar [_]
+  (b/copy-dir
+   {:src-dirs ["src" "resources"]
+    :target-dir class-dir})
+
+  (b/compile-clj
+   {:basis basis
+    :src-dirs ["src"]
+    :class-dir class-dir})
+
+  (b/uber
+   {:class-dir class-dir
+    :uber-file jar-file
+    :main 'clojure.main
+    :exclude [#"goog.*" #"^javasist.*"]
+    :basis basis}))
+
+(defn compile [_]
+  (b/javac
+   {:src-dirs ["dev/java"]
+    :class-dir class-dir
+    :basis basis
+    :javac-opts ["-source" "11" "-target" "11"]}))

backend/deps.edn

@@ -1,78 +1,80 @@
-{
- ;; :mvn/repos
- ;; {"central" {:url "https://repo1.maven.org/maven2/"}
- ;;  "clojars" {:url "https://clojars.org/repo"}
- ;;  "jcenter" {:url "https://jcenter.bintray.com/"}
- ;;  }
- :deps
- {penpot/common
-  {:local/root "../common"}
+{:deps
+ {penpot/common {:local/root "../common"}
+  org.clojure/clojure {:mvn/version "1.11.1"}
+  org.clojure/core.async {:mvn/version "1.5.648"}
 
   ;; Logging
   org.zeromq/jeromq {:mvn/version "0.5.2"}
 
   com.taoensso/nippy {:mvn/version "3.1.1"}
-  com.github.luben/zstd-jni {:mvn/version "1.4.9-5"}
+  com.github.luben/zstd-jni {:mvn/version "1.5.2-3"}
+  org.clojure/data.fressian {:mvn/version "1.0.0"}
 
-  ;; NOTE: don't upgrade to latest version, breaking change is
-  ;; introduced on 0.10.0 that suffixes counters with _total if they
-  ;; are not already has this suffix.
-  io.prometheus/simpleclient {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_hotspot {:mvn/version "0.9.0"}
-  io.prometheus/simpleclient_jetty {:mvn/version "0.9.0"
+  io.prometheus/simpleclient {:mvn/version "0.15.0"}
+  io.prometheus/simpleclient_hotspot {:mvn/version "0.15.0"}
+  io.prometheus/simpleclient_jetty {:mvn/version "0.15.0"
                                     :exclusions [org.eclipse.jetty/jetty-server
                                                  org.eclipse.jetty/jetty-servlet]}
-  io.prometheus/simpleclient_httpserver {:mvn/version "0.9.0"}
+  io.prometheus/simpleclient_httpserver {:mvn/version "0.15.0"}
 
-  io.lettuce/lettuce-core {:mvn/version "6.1.2.RELEASE"}
-  java-http-clj/java-http-clj {:mvn/version "0.4.2"}
+  io.lettuce/lettuce-core {:mvn/version "6.1.8.RELEASE"}
+  java-http-clj/java-http-clj {:mvn/version "0.4.3"}
 
-  info.sunng/ring-jetty9-adapter {:mvn/version "0.15.1"}
-  com.github.seancorfield/next.jdbc {:mvn/version "1.2.659"}
-  metosin/reitit-ring {:mvn/version "0.5.13"}
-  org.postgresql/postgresql {:mvn/version "42.2.20"}
-  com.zaxxer/HikariCP {:mvn/version "4.0.3"}
+  funcool/yetti {:git/tag "v9.8" :git/sha "fbe1d7d"
+                 :git/url "https://github.com/funcool/yetti.git"
+                 :exclusions [org.slf4j/slf4j-api]}
 
-  funcool/datoteka {:mvn/version "2.0.0"}
+  com.github.seancorfield/next.jdbc {:mvn/version "1.2.780"}
+  metosin/reitit-core {:mvn/version "0.5.18"}
+  org.postgresql/postgresql {:mvn/version "42.4.0"}
+  com.zaxxer/HikariCP {:mvn/version "5.0.1"}
 
-  buddy/buddy-core {:mvn/version "1.10.1"}
-  buddy/buddy-hashers {:mvn/version "1.8.1"}
-  buddy/buddy-sign {:mvn/version "3.4.1"}
+  funcool/datoteka {:mvn/version "3.0.64"}
 
-  org.jsoup/jsoup {:mvn/version "1.13.1"}
-  org.im4java/im4java {:mvn/version "1.4.0"}
-  org.lz4/lz4-java {:mvn/version "1.7.1"}
+  buddy/buddy-hashers {:mvn/version "1.8.158"}
+  buddy/buddy-sign {:mvn/version "3.4.333"}
+
+  org.jsoup/jsoup {:mvn/version "1.15.1"}
+  org.im4java/im4java {:git/tag "1.4.0-penpot-2" :git/sha "e2b3e16"
+                       :git/url "https://github.com/penpot/im4java"}
+
+  org.lz4/lz4-java {:mvn/version "1.8.0"}
 
   org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
   integrant/integrant {:mvn/version "0.8.0"}
 
-  software.amazon.awssdk/s3 {:mvn/version "2.16.62"}}
+  io.sentry/sentry {:mvn/version "5.6.1"}
 
- :paths ["src" "resources"]
+  dawran6/emoji {:mvn/version "0.1.5"}
+  markdown-clj/markdown-clj {:mvn/version "1.11.1"}
+
+  ;; Pretty Print specs
+  pretty-spec/pretty-spec {:mvn/version "0.1.4"}
+  software.amazon.awssdk/s3 {:mvn/version "2.17.209"}}
+
+ :paths ["src" "resources" "target/classes"]
  :aliases
  {:dev
   {:extra-deps
    {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
     org.clojure/tools.namespace {:mvn/version "RELEASE"}
     org.clojure/test.check {:mvn/version "RELEASE"}
-    com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.0"}
-
-    criterium/criterium {:mvn/version "RELEASE"}
+    clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
+    org.clojure/data.csv {:mvn/version "RELEASE"}
+    com.clojure-goes-fast/clj-async-profiler {:mvn/version "RELEASE"}
     mockery/mockery {:mvn/version "RELEASE"}}
    :extra-paths ["test" "dev"]}
 
-  :fn-fixtures
-  {:exec-fn app.cli.fixtures/run
-   :args {}}
-
-  :kaocha
-  {:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.829"}}
-   :main-opts ["-m" "kaocha.runner"]}
+  :build
+  {:extra-deps
+   {io.github.clojure/tools.build {:git/tag "v0.8.2" :git/sha "ba1a2bf"}}
+   :ns-default build}
 
   :test
-  {:extra-deps {io.github.cognitect-labs/test-runner
-                {:git/url "https://github.com/cognitect-labs/test-runner.git"
-                 :sha "705ad25bbf0228b1c38d0244a36001c2987d7337"}}
+  {:extra-paths ["test"]
+   :extra-deps
+   {io.github.cognitect-labs/test-runner
+    {:git/tag "v0.5.0" :git/sha "b3fd0d2"}}
    :exec-fn cognitect.test-runner.api/test}
 
   :outdated

backend/dev/script-fix-sobjects.clj

@@ -0,0 +1,114 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+;; This is an example on how it can be executed:
+;; clojure -Scp $(cat classpath) -M dev/script-fix-sobjects.clj
+
+(require
+ '[app.common.logging :as l]
+ '[app.common.data :as d]
+ '[app.common.pprint]
+ '[app.db :as db]
+ '[app.storage :as sto]
+ '[app.storage.impl :as impl]
+ '[app.util.time :as dt]
+ '[integrant.core :as ig])
+
+;; --- HELPERS
+
+(l/info :hint "initializing script" :args *command-line-args*)
+
+(def noop? (some #(= % "noop") *command-line-args*))
+(def chunk-size 10)
+
+(def sql:retrieve-sobjects-chunk
+  "SELECT * FROM storage_object
+    WHERE created_at < ? AND deleted_at is NULL
+    ORDER BY created_at desc LIMIT ?")
+
+(defn get-chunk
+  [conn cursor]
+  (let [rows (db/exec! conn [sql:retrieve-sobjects-chunk cursor chunk-size])]
+    [(some->> rows peek :created-at) (seq rows)]))
+
+(defn get-candidates
+  [conn]
+  (->> (d/iteration (partial get-chunk conn)
+                    :vf second
+                    :kf first
+                    :initk (dt/now))
+       (sequence cat)))
+
+(def modules
+  [:app.db/pool
+   :app.storage/storage
+   [:app.main/default :app.worker/executor]
+   [:app.main/assets :app.storage.s3/backend]
+   [:app.main/assets :app.storage.fs/backend]])
+
+(def system
+  (let [config (select-keys app.main/system-config modules)
+        config (-> config
+                   (assoc :app.migrations/all {})
+                   (assoc :app.metrics/metrics nil))]
+    (ig/load-namespaces config)
+    (-> config ig/prep ig/init)))
+
+(defn update-fn
+  [{:keys [conn] :as storage} {:keys [id backend] :as row}]
+  (cond
+    (= backend "s3")
+    (do
+      (l/info :hint "rename storage object backend"
+              :id id
+              :from-backend backend
+              :to-backend :assets-s3)
+      (assoc row :backend "assets-s3"))
+
+    (= backend "assets-s3")
+    (do
+      (l/info :hint "ignoring storage object" :id id :backend backend)
+      nil)
+
+    (or (= backend "fs")
+        (= backend "assets-fs"))
+    (let [sobj (sto/row->storage-object row)
+          path (-> (sto/get-object-path storage sobj) deref)]
+      (l/info :hint "change storage object backend"
+              :id id
+              :from-backend backend
+              :to-backend :assets-s3)
+      (when-not noop?
+        (-> (impl/resolve-backend storage :assets-s3)
+            (impl/put-object sobj (sto/content path))
+            (deref)))
+      (assoc row :backend "assets-s3"))
+
+    :else
+    (throw (IllegalArgumentException. "unexpected backend found"))))
+
+(try
+  (db/with-atomic [conn (:app.db/pool system)]
+    (let [storage (:app.storage/storage system)
+          storage (assoc storage :conn conn)]
+      (loop [items (get-candidates conn)]
+        (when-let [item (first items)]
+          (when-let [{:keys [id] :as row} (update-fn storage item)]
+          (db/update! conn :storage-object (dissoc row :id) {:id (:id item)}))
+          (recur (rest items))))
+      (when noop?
+        (throw (ex-info "explicit rollback" {})))))
+
+  (catch Throwable cause
+    (cond
+      (= "explicit rollback" (ex-message cause))
+      (l/warn :hint "transaction aborted")
+
+      :else
+      (l/error :hint "unexpected exception" :cause cause))))
+
+(ig/halt! system)
+(System/exit 0)

backend/dev/user.clj

@@ -6,47 +6,35 @@
 
 (ns user
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.geom.matrix :as gmt]
+   [app.common.perf :as perf]
+   [app.common.transit :as t]
    [app.config :as cfg]
    [app.main :as main]
    [app.util.blob :as blob]
+   [app.util.fressian :as fres]
    [app.util.json :as json]
    [app.util.time :as dt]
-   [app.util.transit :as t]
+   [clj-async-profiler.core :as prof]
+   [clojure.contrib.humanize :as hum]
    [clojure.java.io :as io]
    [clojure.pprint :refer [pprint print-table]]
    [clojure.repl :refer :all]
    [clojure.spec.alpha :as s]
    [clojure.spec.gen.alpha :as sgen]
    [clojure.test :as test]
-   [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
-   [criterium.core :refer [quick-bench bench with-progress-reporting]]
+   [datoteka.core]
    [integrant.core :as ig]))
 
 (repl/disable-reload! (find-ns 'integrant.core))
+(set! *warn-on-reflection* true)
 
 (defonce system nil)
 
-;; --- Benchmarking Tools
-
-(defmacro run-quick-bench
-  [& exprs]
-  `(with-progress-reporting (quick-bench (do ~@exprs) :verbose)))
-
-(defmacro run-quick-bench'
-  [& exprs]
-  `(quick-bench (do ~@exprs)))
-
-(defmacro run-bench
-  [& exprs]
-  `(with-progress-reporting (bench (do ~@exprs) :verbose)))
-
-(defmacro run-bench'
-  [& exprs]
-  `(bench (do ~@exprs)))
-
 ;; --- Development Stuff
 
 (defn- run-tests
@@ -91,7 +79,16 @@
 
 (defn compression-bench
   [data]
-  (print-table
-   [{:v1 (alength (blob/encode data {:version 1}))
-     :v2 (alength (blob/encode data {:version 2}))
-     :v3 (alength (blob/encode data {:version 3}))}]))
+  (let [humanize (fn [v] (hum/filesize v :binary true :format " %.4f "))]
+    (print-table
+     [{:v1 (humanize (alength (blob/encode data {:version 1})))
+       :v2 (humanize (alength (blob/encode data {:version 2})))
+       :v3 (humanize (alength (blob/encode data {:version 3})))
+       :v4 (humanize (alength (blob/encode data {:version 4})))
+       }])))
+
+(defonce debug-tap
+  (do
+    (add-tap #(locking debug-tap
+                (prn "tap debug:" %)))
+    1))

backend/resources/api-doc-entry.tmpl

@@ -0,0 +1,54 @@
+<li class="rpc-item">
+  <div class="rpc-row-info">
+    {# <div class="type">{{item.type}}</div> #}
+    <div class="module">{{item.module}}:</div>
+    <div class="name">{{item.name}}</div>
+    <div class="tags">
+      {% if item.deprecated %}
+        <span class="tag">
+          <span>Deprecated:</span>
+          <span>since v{{item.deprecated}}</span>,
+        </span>
+      {% endif %}
+      <span class="tag">
+        <span>Auth:</span>
+        <span>{% if item.auth %}YES{% else %}NO{% endif %}</span>
+      </span>
+    </div>
+  </div>
+  <div class="rpc-row-detail hidden">
+    <h3>DOCSTRING:</h3>
+
+    <section class="padded-section">
+
+      {% if item.added %}
+        <p class="small"><strong>Added:</strong> on v{{item.added}}</p>
+      {% endif %}
+
+      {% if item.deprecated %}
+        <p class="small"><strong>Deprecated:</strong> since v{{item.deprecated}}</p>
+      {% endif %}
+
+      {% if item.docs %}
+        <p class="docstring"> {{item.docs}}</p>
+      {% endif %}
+    </section>
+
+    {% if item.changes %}
+      <h3>CHANGES:</h3>
+      <section class="padded-section">
+
+        <ul class="changes">
+          {% for change in item.changes %}
+            <li><strong>{{change.0}}</strong> - {{change.1}}</li>
+          {% endfor %}
+        </ul>
+      </section>
+    {% endif %}
+
+    <h3>SPEC EXPLAIN:</h3>
+    <section class="padded-section">
+      <pre class="spec-explain">{{item.spec}}</pre>
+    </section>
+  </div>
+</li>

backend/resources/api-doc.css

@@ -0,0 +1,145 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+pre {
+  margin: 0px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+  padding-top: 20px;
+  padding-bottom: 20px;
+  display: flex;
+  justify-content: center;
+}
+
+main {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  min-width: 900px;
+  width: 900px;
+}
+
+header {
+  border-bottom: 1px solid #c0c0c0;
+  display: flex;
+  justify-content: center;
+  width: 100%;
+}
+
+.rpc-doc-content {
+  margin-top: 20px;
+  width: 100%;
+  display: flex;
+  flex-direction: column;
+  /* border: 1px solid red; */
+  padding: 5px;
+}
+
+.rpc-doc-content > h2:not(:first-child) {
+  margin-top: 30px;
+}
+
+
+.rpc-items {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+}
+
+.rpc-item {
+  /* border: 1px solid red; */
+  /* cursor: pointer; */
+  display: flex;
+  flex-direction: column;
+}
+
+.rpc-item:not(:last-child) {
+  margin-bottom: 3px;
+}
+
+.rpc-row-info {
+  cursor: pointer;
+  display: flex;
+  background-color: #eeeeee;
+  padding: 5px 10px;
+}
+
+.rpc-row-info > *:not(:last-child) {
+  margin-right: 10px;
+}
+
+.rpc-row-info > * {
+  /* border: 1px solid green; */
+}
+
+.rpc-row-info > .type {
+  font-weight: bold;
+  width: 70px;
+}
+
+.rpc-row-info > .name {
+  width: 280px;
+  /* font-weight: bold; */
+  border-right: 1px dotted #777;
+  padding-right: 10px;
+}
+
+.rpc-row-info > .module {
+  width: 120px;
+  font-weight: bold;
+  border-right: 1px dotted #777;
+  text-align: right;
+  padding-right: 10px;
+}
+
+.rpc-row-info > .tags > .tag > span:first-child {
+  font-weight: bold;
+}
+
+.hidden {
+  display: none;
+}
+
+.rpc-row-detail {
+  padding: 5px 10px;
+  padding-bottom: 20px;
+}
+
+.rpc-row-detail p {
+  font-weight: 200;
+}
+
+.rpc-row-detail p.small {
+  margin-top: 2px;
+  margin-bottom: 2px;
+  font-size: 10px;
+}
+
+.rpc-row-detail p.small {
+  margin-top: 2px;
+  margin-bottom: 2px;
+  font-size: 10px;
+}
+
+.rpc-row-detail strong {
+  font-weight: 500;
+}
+
+.rpc-row-detail .changes {
+  font-weight: 200;
+  list-style: none;
+  padding: 0px;
+}
+
+.rpc-row-detail .padded-section {
+  padding: 0px 10px;
+}
+
+p.small strong {
+  font-size: 10px;
+}

backend/resources/api-doc.js

@@ -0,0 +1,27 @@
+(function() {
+  document.addEventListener("DOMContentLoaded", function(event) {
+    const rows = document.querySelectorAll(".rpc-row-info");
+
+    const onRowClick = (event) => {
+      const target = event.currentTarget;
+      for (let node of rows) {
+        if (node !== target) {
+          node.nextElementSibling.classList.add("hidden");
+        } else {
+          const sibling = target.nextElementSibling;
+
+          if (sibling.classList.contains("hidden")) {
+            sibling.classList.remove("hidden");
+          } else {
+            sibling.classList.add("hidden");
+          }
+        }
+      }
+    };
+
+    for (let node of rows) {
+      node.addEventListener("click", onRowClick);
+    }
+
+  });
+})();

backend/resources/api-doc.tmpl

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Builtin API Documentation - Penpot</title>
+
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+    <link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@200;300;400;500;700&display=swap" rel="stylesheet">
+    <style>
+      {% include "api-doc.css" %}
+    </style>
+    <script>
+      {% include "api-doc.js" %}
+    </script>
+  </head>
+  <body>
+    <main>
+      <header>
+        <h1>Penpot API Documentation (v{{version}})</h1>
+      </header>
+      <section class="rpc-doc-content">
+
+        <h2>RPC COMMAND METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in command-methods %}
+            {% include "api-doc-entry.tmpl" with item=item %}
+          {% endfor %}
+        </ul>
+
+        <h2>RPC QUERY METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in query-methods %}
+            {% include "api-doc-entry.tmpl" with item=item %}
+          {% endfor %}
+        </ul>
+
+        <h2>RPC MUTATION METHODS:</h2>
+        <ul class="rpc-items">
+          {% for item in mutation-methods %}
+            {% include "api-doc-entry.tmpl" with item=item %}
+          {% endfor %}
+        </ul>
+      </section>
+    </main>
+  </body>
+</html>
+

backend/resources/emails-mjml/change-email/en.mjml

@@ -37,7 +37,7 @@
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/invite-to-team/en.mjml

@@ -30,7 +30,7 @@
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/password-recovery/en.mjml

@@ -39,7 +39,7 @@
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/register/en.mjml

@@ -22,7 +22,7 @@
         <mj-text font-size="24px" font-weight="600">Hello {{name}}!</mj-text>
         <mj-text>
           Thanks for signing up for your Penpot account! Please verify your
-          email using the link below adn get started building mockups and
+          email using the link below and get started building mockups and
           prototypes today!
         </mj-text>
         <mj-button href="{{ public-uri }}/#/auth/verify-token?token={{token}}">
@@ -36,7 +36,7 @@
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails/change-email/en.html

@@ -250,7 +250,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.</div>
                     </td>
                   </tr>
                 </table>
@@ -450,7 +450,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/feedback/en.subj

@@ -1 +1 @@
-[PENPOT FEEDBACK]: {{subject|abbreviate:19}} (from {{email}})
+[PENPOT FEEDBACK]: {{subject}}

backend/resources/emails/invite-to-team/en.html

@@ -240,7 +240,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.</div>
                     </td>
                   </tr>
                 </table>
@@ -440,7 +440,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/password-recovery/en.html

@@ -245,7 +245,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.</div>
                     </td>
                   </tr>
                 </table>
@@ -445,7 +445,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/register/en.html

@@ -173,7 +173,7 @@
                   </tr>
                   <tr>
                     <td align="left" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below adn get started building mockups and prototypes today!</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:16px;line-height:150%;text-align:left;color:#000000;">Thanks for signing up for your Penpot account! Please verify your email using the link below and get started building mockups and prototypes today!</div>
                     </td>
                   </tr>
                   <tr>
@@ -240,7 +240,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot is the first Open Source design and prototyping platform meant for cross-domain teams.</div>
                     </td>
                   </tr>
                 </table>
@@ -440,7 +440,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>
@@ -465,4 +465,4 @@
   </div>
 </body>
 
-</html>
+</html>

backend/resources/emails/register/en.txt

@@ -1,7 +1,7 @@
 Hello {{name}}!
 
 Thanks for signing up for your Penpot account! Please verify your email using the
-link below adn get started building mockups and prototypes today!
+link below and get started building mockups and prototypes today!
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 

backend/resources/error-report.tmpl

@@ -1,194 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <meta http-equiv="x-ua-compatible" content="ie=edge" />
-    <title>penpot - error report {{id}}</title>
-    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
-    <style>
-      body {
-        margin: 0px;
-        padding: 0px;
-      }
-      pre {
-        margin: 0px;
-      }
-      * {
-        font-family: "JetBrains Mono", monospace; 
-        font-size: 12px;
-      }
-      .table {
-        display: flex;
-        flex-direction: column;
-        margin: 10px;
-      }
-
-      .table-row {
-        display: flex;
-        /* width: 100%; */
-        /* border: 1px solid red; */
-      }
-
-      .table-key {
-        font-weight: 600;
-        width: 60px;
-        padding: 4px;
-      }
-
-      .table-val {
-        font-weight: 200;
-        color: #333;
-        padding: 4px;
-      }
-
-      .multiline {
-        margin-top: 15px;
-        flex-direction: column;
-      }
-
-      .multiline .table-key {
-        margin-bottom: 10px;
-        border-bottom: 1px dashed #dddddd;
-        /* padding: 4px; */
-        width: unset;
-      }
-
-    </style>
-  </head>
-  <body>
-    <div class="table">
-      <div class="table-row">
-        <div class="table-key" title="Error ID">ERID: </div>
-        <div class="table-val">{{id}}</div>
-      </div>
-      {% if profile-id %}
-      <div class="table-row">
-        <div class="table-key" title="Profile ID">PFID: </div>
-        <div class="table-val">{{profile-id}}</div>
-      </div>
-      {% endif %}
-
-      {% if user-agent %}
-      <div class="table-row">
-        <div class="table-key">UAGT: </div>
-        <div class="table-val">{{user-agent}}</div>
-      </div>
-      {% endif %}
-
-      {% if frontend-version %}
-      <div class="table-row">
-        <div class="table-key">FVER: </div>
-        <div class="table-val">{{frontend-version}}</div>
-      </div>
-      {% endif %}
-
-      <div class="table-row">
-        <div class="table-key">BVER: </div>
-        <div class="table-val">{{version}}</div>
-      </div>
-
-      {% if host %}
-      <div class="table-row">
-        <div class="table-key">HOST: </div>
-        <div class="table-val">{{host}}</div>
-      </div>
-      {% endif %}
-
-      {% if tenant %}
-      <div class="table-row">
-        <div class="table-key">ENV: </div>
-        <div class="table-val">{{tenant}}</div>
-      </div>
-      {% endif %}
-
-      {% if public-uri %}
-      <div class="table-row">
-        <div class="table-key">PURI: </div>
-        <div class="table-val">{{public-uri}}</div>
-      </div>
-      {% endif %}
-
-      {% if type %}
-      <div class="table-row">
-        <div class="table-key">TYPE: </div>
-        <div class="table-val">{{type}}</div>
-      </div>
-      {% endif %}
-
-      {% if code %}
-      <div class="table-row">
-        <div class="table-key">CODE: </div>
-        <div class="table-val">{{code}}</div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div class="table-row">
-        <div class="table-key">CLSS: </div>
-        <div class="table-val">{{error.class}}</div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div class="table-row">
-        <div class="table-key">HINT: </div>
-        <div class="table-val">{{error.message}}</div>
-      </div>
-      {% endif %}
-
-      {% if method %}
-      <div class="table-row">
-        <div class="table-key">PATH: </div>
-        <div class="table-val">{{method|upper}} {{path}}</div>
-      </div>
-      {% endif %}
-
-      {% if explain %}
-        <div>(<a href="#explain">go to explain</a>)</div>
-      {% endif %}
-      {% if data %}
-        <div>(<a href="#edata">go to edata</a>)</div>
-      {% endif %}
-      {% if error %}
-        <div>(<a href="#trace">go to trace</a>)</div>
-      {% endif %}
-
-      {% if params %}
-      <div id="params" class="table-row multiline">
-        <div class="table-key">PARAMS: </div>
-        <div class="table-val">
-          <pre>{{params}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if explain %}
-      <div id="explain" class="table-row multiline">
-        <div class="table-key">EXPLAIN: </div>
-        <div class="table-val">
-          <pre>{{explain}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if data %}
-      <div id="edata" class="table-row multiline">
-        <div class="table-key">EDATA: </div>
-        <div class="table-val">
-          <pre>{{data}}</pre>
-        </div>
-      </div>
-      {% endif %}
-
-      {% if error %}
-      <div id="trace" class="table-row multiline">
-        <div class="table-key">TRACE:</div>
-        <div class="table-val">
-          <pre>{{error.trace}}</pre>
-        </div>
-      </div>
-      {% endif %}
-    </div>
-  </body>
-</html>
-

backend/resources/log4j2-devenv.xml

@@ -2,7 +2,7 @@
 <Configuration status="info" monitorInterval="30">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
 
     <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
@@ -20,11 +20,17 @@
   </Appenders>
 
   <Loggers>
-    <Logger name="com.zaxxer.hikari" level="error"/>
     <Logger name="io.lettuce" level="error" />
-    <Logger name="org.eclipse.jetty" level="error" />
+    <Logger name="com.zaxxer.hikari" level="error"/>
     <Logger name="org.postgresql" level="error" />
 
+    <Logger name="app.rpc.commands.binfile" level="debug" />
+    <Logger name="app.storage.tmp" level="info" />
+    <Logger name="app.worker" level="info" />
+    <Logger name="app.msgbus" level="info" />
+    <Logger name="app.http.websocket" level="info" />
+    <Logger name="app.util.websocket" level="info" />
+
     <Logger name="app.cli" level="debug" additivity="false">
       <AppenderRef ref="console"/>
     </Logger>
@@ -38,11 +44,6 @@
       <AppenderRef ref="zmq" level="debug" />
     </Logger>
 
-    <Logger name="penpot" level="debug" additivity="false">
-      <AppenderRef ref="main" level="debug" />
-      <AppenderRef ref="zmq" level="debug" />
-    </Logger>
-
     <Logger name="user" level="trace" additivity="false">
       <AppenderRef ref="main" level="trace" />
     </Logger>

backend/resources/log4j2.xml

@@ -2,19 +2,16 @@
 <Configuration status="info" monitorInterval="60">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
   </Appenders>
 
   <Loggers>
+    <Logger name="io.lettuce" level="error" />
     <Logger name="com.zaxxer.hikari" level="error" />
-    <Logger name="org.eclipse.jetty" level="error" />
+    <Logger name="org.postgresql" level="error" />
 
-    <Logger name="app" level="debug" additivity="false">
-      <AppenderRef ref="console" />
-    </Logger>
-
-    <Logger name="penpot" level="fatal" additivity="false">
+    <Logger name="app" level="info" additivity="false">
       <AppenderRef ref="console" />
     </Logger>
 

backend/resources/templates/base.tmpl

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="robots" content="noindex,nofollow">
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>{% block title %}{% endblock %}</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      {% include "templates/styles.css"  %}
+    </style>
+  </head>
+  <body>
+    {% block content %}
+    {% endblock %}
+  </body>
+</html>
+

backend/resources/templates/debug.tmpl

@@ -0,0 +1,127 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+Debug Main Page
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Debug INDEX:</h1>
+  <div>[<a href="/dbg/error">ERRORS</a>]</div>
+</nav>
+<main class="index">
+  <section class="widget">
+    <fieldset>
+      <legend>Download file data:</legend>
+      <desc>Given an FILE-ID, downloads the file data as file. The file data is encoded using transit.</desc>
+      <form method="get" action="/dbg/file/data">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="submit" name="download" value="Download" />
+          <input type="submit" name="clone" value="Clone" />
+        </div>
+      </form>
+    </fieldset>
+
+    <fieldset>
+      <legend>Upload File Data:</legend>
+      <desc>Create a new file on your draft projects using the file downloaded from the previous section.</desc>
+      <form method="post" enctype="multipart/form-data" action="/dbg/file/data">
+        <div class="row">
+          <input type="file" name="file" value="" />
+        </div>
+        <div class="row">
+          <label>Import with same id?</label>
+          <input type="checkbox" name="reuseid" />
+        </div>
+
+        <input type="submit" value="Upload" />
+      </form>
+    </fieldset>
+  </section>
+
+  <section class="widget">
+    <fieldset>
+      <legend>Export binfile:</legend>
+      <desc>Given an FILE-ID, downloads the file and optionally all
+      the related libraries in a single custom formatted binary
+      file.</desc>
+
+      <form method="get" action="/dbg/file/export">
+        <div class="row set-of-inputs">
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+          <input type="text" style="width:300px" name="file-ids" placeholder="file-id" />
+        </div>
+
+        <div class="row">
+          <label>Include libraries?</label>
+          <input type="checkbox" name="includelibs" />
+        </div>
+
+        <div class="row">
+          <label>Embed assets?</label>
+          <input type="checkbox" name="embedassets" checked/>
+        </div>
+
+        <div class="row">
+          <input type="submit" name="download" value="Download" />
+          <input type="submit" name="clone" value="Clone" />
+        </div>
+      </form>
+    </fieldset>
+    <fieldset>
+      <legend>Import binfile:</legend>
+      <desc>Import penpot file in binary
+      format. If <strong>overwrite</strong> is checked, all files will
+      be overwriten using the same ids found in the file instead of
+      generating a new ones.</desc>
+
+      <form method="post" enctype="multipart/form-data" action="/dbg/file/import">
+        <div class="row">
+          <input type="file" name="file" value="" />
+        </div>
+
+        <div class="row">
+          <label>Overwrite?</label>
+          <input type="checkbox" name="overwrite" />
+          <br />
+          <small>
+            Instead of creating a new file with all relations remaped,
+            reuses all ids and updates/overwrites the objects that are
+            already exists on the database.
+            <strong>Warning, this operation should be used with caution.</strong>
+          </small>
+        </div>
+
+        <div class="row">
+          <label>Migrate?</label>
+          <input type="checkbox" name="migrate" />
+          <br />
+          <small>
+            Applies the file migrations on the importation process.
+          </small>
+        </div>
+
+        <div class="row">
+          <label>Ignore index errors?</label>
+          <input type="checkbox" name="ignore-index-errors" checked/>
+          <br />
+          <small>
+            Do not break on index lookup erros (remap operation).
+            Useful when importing a broken file that has broken
+            relations or missing pieces.
+          </small>
+        </div>
+
+        <div class="row">
+          <input type="submit" name="upload" value="Upload" />
+        </div>
+      </form>
+    </fieldset>
+  </section>
+</main>
+{% endblock %}

backend/resources/templates/error-list.tmpl

@@ -0,0 +1,18 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error list
+{% endblock %}
+
+{% block content %}
+<nav>
+  <h1>Latest error reports:</h1>
+</nav>
+<main class="horizontal-list">
+  <ul>
+    {% for item in items %}
+    <li><a href="/dbg/error/{{item.id}}">{{item.created-at}}</a></li>
+    {% endfor %}
+  </ul>
+</main>
+{% endblock %}

backend/resources/templates/error-report.tmpl

@@ -0,0 +1,98 @@
+{% extends "templates/base.tmpl" %}
+
+{% block title %}
+penpot - error report {{id}}
+{% endblock %}
+
+{% block content %}
+<nav>
+  <div>[<a href="/dbg/error">⮜</a>]</div>
+  <div>[<a href="#context">context</a>]</div>
+  <div>[<a href="#params">request params</a>]</div>
+  {% if data %}
+  <div>[<a href="#edata">error data</a>]</div>
+  {% endif %}
+  {% if spec-explain %}
+  <div>[<a href="#spec-explain">spec explain</a>]</div>
+  {% endif %}
+  {% if spec-problems %}
+  <div>[<a href="#spec-problems">spec problems</a>]</div>
+  {% endif %}
+  {% if spec-value %}
+  <div>[<a href="#spec-value">spec value</a>]</div>
+  {% endif %}
+
+  {% if trace %}
+  <div>[<a href="#trace">error trace</a>]</div>
+  {% endif %}
+</nav>
+<main>
+  <div class="table">
+    <div class="table-row multiline">
+      <div id="context" class="table-key">CONTEXT: </div>
+
+      <div class="table-val">
+        <h1>{{hint}}</h1>
+      </div>
+
+      <div class="table-val">
+        <pre>{{context}}</pre>
+      </div>
+    </div>
+
+    {% if params %}
+    <div class="table-row multiline">
+      <div id="params" class="table-key">REQUEST PARAMS: </div>
+      <div class="table-val">
+        <pre>{{params}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if data %}
+    <div class="table-row multiline">
+      <div id="edata" class="table-key">ERROR DATA: </div>
+      <div class="table-val">
+        <pre>{{data}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-explain %}
+    <div class="table-row multiline">
+      <div id="spec-explain" class="table-key">SPEC EXPLAIN: </div>
+      <div class="table-val">
+        <pre>{{spec-explain}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-problems %}
+    <div class="table-row multiline">
+      <div id="spec-problems" class="table-key">SPEC PROBLEMS: </div>
+      <div class="table-val">
+        <pre>{{spec-problems}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if spec-value %}
+    <div class="table-row multiline">
+      <div id="spec-value" class="table-key">SPEC VALUE: </div>
+      <div class="table-val">
+        <pre>{{spec-value}}</pre>
+      </div>
+    </div>
+    {% endif %}
+
+    {% if trace %}
+    <div class="table-row multiline">
+      <div id="trace" class="table-key">TRACE:</div>
+      <div class="table-val">
+        <pre>{{trace}}</pre>
+      </div>
+    </div>
+    {% endif %}
+  </div>
+</main>
+{% endblock %}

backend/resources/templates/styles.css

@@ -0,0 +1,179 @@
+* {
+  font-family: "JetBrains Mono", monospace;
+  font-size: 12px;
+}
+
+body {
+  margin: 0px;
+  padding: 0px;
+}
+
+pre {
+  margin: 0px;
+  line-height: 16px;
+}
+
+desc {
+  margin-bottom: 10px;
+  font-size: 10px;
+  color: #666;
+}
+
+input[type=text], input[type=submit] {
+  padding: 3px;
+}
+
+main {
+  margin: 20px;
+}
+
+small {
+  font-size: 9px;
+  color: #888;
+}
+
+small > strong {
+  font-size: 9px;
+}
+
+nav {
+  position: fixed;
+  width: 100vw;
+  top: 0;
+  left: 0;
+  padding: 5px 20px;
+  display: flex;
+  background: #e3e3e3;
+}
+
+nav > h1 {
+  padding: 0px;
+  margin: 0px;
+  font-size: 11px;
+}
+
+nav > div {
+  text-transform: uppercase;
+  font-weight: bold;
+}
+
+nav > div:not(:last-child) {
+  margin-right: 10px;
+}
+
+.table {
+  margin-top: 25px;
+  display: flex;
+  flex-direction: column;
+}
+
+.table-row {
+  display: flex;
+  padding-bottom: 15px;
+  /* width: 100%; */
+  /* border: 1px solid red; */
+}
+
+.table-key {
+  font-weight: 600;
+  width: 60px;
+  padding: 4px;
+
+  padding-top: 40px;
+  margin-top: -40px;
+}
+
+.table-val {
+  font-weight: 200;
+  color: #333;
+  padding: 4px;
+}
+
+.multiline {
+  margin-top: 15px;
+  flex-direction: column;
+}
+
+.multiline .table-key {
+  margin-bottom: 10px;
+  border-bottom: 1px dashed #dddddd;
+  /* padding: 4px; */
+  width: unset;
+}
+
+.index {
+  margin-top: 40px;
+  display: flex;
+}
+
+.index > section {
+  padding: 10px;
+  background-color: #e3e3e3;
+  max-width: 400px;
+  margin: 5px;
+  height: fit-content;
+}
+
+.index fieldset:not(:first-child) {
+  margin-top: 15px;
+}
+
+/* .index > section:not(:last-child) { */
+/*   margin-bottom: 10px; */
+/* } */
+
+
+.index > section > h2 {
+  margin-top: 0px;
+}
+
+.horizontal-list {
+  margin: 20px;
+  margin-top: 40px;
+}
+
+.horizontal-list ul {
+  display: flex;
+  margin: 0px;
+  padding: 0px;
+  flex-direction: column;
+  flex-wrap: wrap;
+  height: calc(100vh - 75px);
+  justify-content: flex-start;
+}
+
+.horizontal-list li {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+  line-height: 18px;
+  min-width: 210px;
+  margin: 0px 20px;
+  cursor: pointer;
+  display: flex;
+  justify-content: center;
+  border-radius: 3px;
+}
+
+.horizontal-list li:hover {
+  background-color: #e9e9e9;
+}
+
+.horizontal-list li > a {
+  text-decoration: none;
+  color: inherit;
+}
+
+form .row {
+  padding: 5px 0;
+}
+
+.set-of-inputs {
+  flex-direction: column;
+  display: flex;
+}
+
+.set-of-inputs input:not(:last-child) {
+  margin-bottom: 3px;
+}
+

backend/scripts/build

@@ -1,79 +1,21 @@
-#!/usr/bin/env bb
+#!/usr/bin/env bash
 
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
+CURRENT_VERSION=$1;
 
-(ns build
-  (:require
-   [clojure.string :as str]
-   [clojure.java.io :as io]
-   [clojure.pprint :refer [pprint]]
-   [babashka.fs :as fs]
-   [babashka.process :refer [$ check]]))
+set -ex
 
-(defn split-cp
-  [data]
-  (str/split data #":"))
+rm -rf target;
+mkdir -p target/classes;
+mkdir -p target/dist;
+echo "$CURRENT_VERSION" > target/classes/version.txt;
+cp ../CHANGES.md target/classes/changelog.md;
 
-(def classpath
-  (->> ($ clojure -Spath)
-       (check)
-       (:out)
-       (slurp)
-       (split-cp)
-       (map str/trim)))
+clojure -T:build jar;
+mv target/penpot.jar target/dist/penpot.jar
+cp scripts/run.template.sh target/dist/run.sh;
+cp scripts/manage.template.sh target/dist/manage.sh;
+chmod +x target/dist/run.sh;
+chmod +x target/dist/manage.sh;
 
-(def classpath-jars
-  (let [xfm (filter #(str/ends-with? % ".jar"))]
-    (into #{} xfm classpath)))
 
-(def classpath-paths
-  (let [xfm (comp (remove #(str/ends-with? % ".jar"))
-                  (filter #(.isDirectory (io/file %))))]
-    (into #{} xfm classpath)))
 
-(def version
-  (or (first *command-line-args*) "%version%"))
-
-;; Clean previous dist
-(-> ($ rm -rf "./target/dist") check)
-
-;; Create a new dist
-(-> ($ mkdir -p "./target/dist/deps") check)
-
-;; Copy all jar deps into dist
-(run! (fn [item] (-> ($ cp ~item "./target/dist/deps/") check)) classpath-jars)
-
-;; Create the application jar
-(spit "./target/dist/version.txt" version)
-
-(-> ($ jar cvf "./target/dist/deps/app.jar" -C ~(first classpath-paths) ".") check)
-(-> ($ jar uvf "./target/dist/deps/app.jar" -C "./target/dist" "version.txt") check)
-(run! (fn [item]
-        (-> ($ jar uvf "./target/dist/deps/app.jar" -C ~item ".")  check))
-      (rest classpath-paths))
-
-;; Copy logging configuration
-(-> ($ cp "./resources/log4j2.xml" "./target/dist/") check)
-
-;; Create classpath file
-(let [jars (->> (into ["app.jar"] classpath-jars)
-                (map fs/file-name)
-                (map #(fs/path "deps" %))
-                (map str))]
-  (spit "./target/dist/classpath" (str/join ":" jars)))
-
-;; Copy run script template
-(-> ($ cp "./scripts/run.template.sh" "./target/dist/run.sh") check)
-
-;; Copy run script template
-(-> ($ cp "./scripts/manage.template.sh" "./target/dist/manage.sh") check)
-
-;; Add exec permisions to scripts.
-(-> ($ chmod +x "./target/dist/run.sh") check)
-(-> ($ chmod +x "./target/dist/manage.sh") check)
-
-nil

backend/scripts/manage.template.sh

@@ -16,4 +16,4 @@ if [ -f ./environ ]; then
    source ./environ
 fi
 
-exec $JAVA_CMD $JVM_OPTS -classpath $(cat classpath) -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "$@"
+exec $JAVA_CMD $JVM_OPTS -jar penpot.jar -m app.cli.manage "$@"

backend/scripts/repl

@@ -1,10 +1,46 @@
 #!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+export PENPOT_HOST=devenv
+export PENPOT_TENANT=dev
+export PENPOT_FLAGS="$PENPOT_FLAGS enable-backend-asserts enable-audit-log enable-transit-readable-response enable-demo-users disable-secure-session-cookies"
 
-export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-Djdk.attach.allowAttachSelf -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
-# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";
-# export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot"
+# export PENPOT_DATABASE_USERNAME="penpot"
+# export PENPOT_DATABASE_PASSWORD="penpot"
+# export PENPOT_DATABASE_READONLY=true
+
+# export PENPOT_DATABASE_URI="postgresql://172.17.0.1:5432/penpot_pre"
+# export PENPOT_DATABASE_USERNAME="penpot_pre"
+# export PENPOT_DATABASE_PASSWORD="penpot_pre"
+
+# export PENPOT_LOGGERS_LOKI_URI="http://172.17.0.1:3100/loki/api/v1/push"
+# export PENPOT_AUDIT_LOG_ARCHIVE_URI="http://localhost:6070/api/audit"
+
+# Initialize MINIO config
+mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
+mc admin user add penpot-s3 penpot-devenv penpot-devenv
+mc admin policy set penpot-s3 readwrite user=penpot-devenv
+mc mb penpot-s3/penpot -p
+
+export AWS_ACCESS_KEY_ID=penpot-devenv
+export AWS_SECRET_ACCESS_KEY=penpot-devenv
+export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
+export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
+export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
+
+export OPTIONS="
+       -A:dev:jmx-remote \
+       -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
+       -J-Dlog4j2.configurationFile=log4j2-devenv.xml \
+       -J-XX:+UseG1GC \
+       -J-XX:-OmitStackTraceInFastThrow \
+       -J-Xms50m -J-Xmx1024m \
+       -J-Djdk.attach.allowAttachSelf \
+       -J-XX:+UnlockDiagnosticVMOptions \
+       -J-XX:+DebugNonSafepoints";
+
+# Uncomment for use the ImageMagick v7.x
+# export OPTIONS="-J-Dim4java.useV7=true $OPTIONS";
 
 export OPTIONS_EVAL="nil"
 # export OPTIONS_EVAL="(set! *warn-on-reflection* true)"

backend/scripts/run.template.sh

@@ -17,4 +17,4 @@ if [ -f ./environ ]; then
 fi
 
 set -x
-exec $JAVA_CMD $JVM_OPTS -classpath "$(cat classpath)" -Dlog4j2.configurationFile=./log4j2.xml "$@" clojure.main -m app.main
+exec $JAVA_CMD $JVM_OPTS "$@" -jar penpot.jar -m app.main

backend/scripts/start-dev

@@ -1,15 +1,25 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-export PENPOT_ASSERTS_ENABLED=true
+export PENPOT_HOST=devenv
+export PENPOT_TENANT=dev
+export PENPOT_FLAGS="$PENPOT_FLAGS enable-backend-asserts enable-audit-log enable-transit-readable-response enable-demo-users disable-secure-session-cookies"
 
 set -ex
 
-if [ ! -e ~/.fixtures-loaded ]; then
-    echo "Loading fixtures..."
-    clojure -Adev -X:fn-fixtures
-    touch ~/.fixtures-loaded
+if [ "$1" = "--watch" ]; then
+    echo "Start Watch..."
+
+    clojure -A:dev -M -m app.main &
+    PID=$!
+
+    npx nodemon \
+        --watch src \
+        --watch ../common \
+        --ext "clj" \
+        --signal SIGKILL \
+        --exec 'echo "(user/restart)" | nc -N localhost 6062'
+
+    kill -9 $PID
+else
+    clojure -A:dev -M -m app.main
 fi
-
-clojure -A:dev -M -m app.main
-
-

backend/src/app/auth/ldap.clj

@@ -0,0 +1,137 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.auth.ldap
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.config :as cf]
+   [clj-ldap.client :as ldap]
+   [clojure.spec.alpha :as s]
+   [clojure.string]
+   [integrant.core :as ig]))
+
+(defn- prepare-params
+  [cfg]
+  {:ssl?      (:ssl cfg)
+   :startTLS? (:tls cfg)
+   :bind-dn   (:bind-dn cfg)
+   :password  (:bind-password cfg)
+   :host      {:address (:host cfg)
+               :port    (:port cfg)}})
+
+(defn- connect
+  "Connects to the LDAP provider and returns a connection. An
+  exception is raised if no connection is possible."
+  ^java.lang.AutoCloseable
+  [cfg]
+  (try
+    (-> cfg prepare-params ldap/connect)
+    (catch Throwable cause
+      (ex/raise :type :restriction
+                :code :unable-to-connect-to-ldap
+                :hint "unable to connect to ldap server"
+                :cause cause))))
+
+(defn- replace-several [s & {:as replacements}]
+  (reduce-kv clojure.string/replace s replacements))
+
+(defn- search-user
+  [{:keys [conn attrs base-dn] :as cfg} email]
+  (let [query   (replace-several (:query cfg) ":username" email)
+        params  {:filter query
+                 :sizelimit 1
+                 :attributes attrs}]
+    (first (ldap/search conn base-dn params))))
+
+(defn- retrieve-user
+  [{:keys [conn] :as cfg} {:keys [email password]}]
+  (when-let [{:keys [dn] :as user} (search-user cfg email)]
+    (when (ldap/bind? conn dn password)
+      {:fullname (get user (-> cfg :attrs-fullname keyword))
+       :email    email
+       :backend  "ldap"})))
+
+(s/def ::fullname ::us/not-empty-string)
+(s/def ::email ::us/email)
+(s/def ::backend ::us/not-empty-string)
+
+(s/def ::info-data
+  (s/keys :req-un [::fullname ::email ::backend]))
+
+(defn authenticate
+  [cfg params]
+  (with-open [conn (connect cfg)]
+    (when-let [user (-> (assoc cfg :conn conn)
+                        (retrieve-user params))]
+      (when-not (s/valid? ::info-data user)
+        (let [explain (s/explain-str ::info-data user)]
+          (l/warn ::l/raw (str "invalid response from ldap, looks like ldap is not configured correctly\n" explain))
+          (ex/raise :type :restriction
+                    :code :wrong-ldap-response
+                    :explain explain)))
+      user)))
+
+(defn- try-connectivity
+  [cfg]
+  ;; If we have ldap parameters, try to establish connection
+  (when (and (:bind-dn cfg)
+             (:bind-password cfg)
+             (:host cfg)
+             (:port cfg))
+    (try
+      (with-open [_ (connect cfg)]
+        (l/info :hint "provider initialized"
+                :provider "ldap"
+                :host (:host cfg)
+                :port (:port cfg)
+                :tls? (:tls cfg)
+                :ssl? (:ssl cfg)
+                :bind-dn (:bind-dn cfg)
+                :base-dn (:base-dn cfg)
+                :query (:query cfg))
+        cfg)
+      (catch Throwable cause
+        (l/error :hint "unable to connect to LDAP server (LDAP auth provider disabled)"
+                 :host (:host cfg) :port (:port cfg) :cause cause)
+        nil))))
+
+(defn- prepare-attributes
+  [cfg]
+  (assoc cfg :attrs [(:attrs-username cfg)
+                     (:attrs-email cfg)
+                     (:attrs-fullname cfg)]))
+
+(defmethod ig/init-key ::provider
+  [_ cfg]
+  (when (:enabled? cfg)
+    (some-> cfg try-connectivity prepare-attributes)))
+
+(s/def ::enabled? ::us/boolean)
+(s/def ::host ::cf/ldap-host)
+(s/def ::port ::cf/ldap-port)
+(s/def ::ssl ::cf/ldap-ssl)
+(s/def ::tls ::cf/ldap-starttls)
+(s/def ::query ::cf/ldap-user-query)
+(s/def ::base-dn ::cf/ldap-base-dn)
+(s/def ::bind-dn ::cf/ldap-bind-dn)
+(s/def ::bind-password ::cf/ldap-bind-password)
+(s/def ::attrs-email ::cf/ldap-attrs-email)
+(s/def ::attrs-fullname ::cf/ldap-attrs-fullname)
+(s/def ::attrs-username ::cf/ldap-attrs-username)
+
+(defmethod ig/pre-init-spec ::provider
+  [_]
+  (s/keys :opt-un [::host ::port
+                   ::ssl ::tls
+                   ::enabled?
+                   ::bind-dn
+                   ::bind-password
+                   ::query
+                   ::attrs-email
+                   ::attrs-username
+                   ::attrs-fullname]))

backend/src/app/auth/oidc.clj

@@ -0,0 +1,538 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.auth.oidc
+  "OIDC client implementation."
+  (:require
+   [app.common.data :as d]
+   [app.common.data.macros :as dm]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [app.common.uri :as u]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.http.middleware :as hmw]
+   [app.loggers.audit :as audit]
+   [app.rpc.queries.profile :as profile]
+   [app.util.json :as json]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.set :as set]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.response :as yrs]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- obfuscate-string
+  [s]
+  (if (< (count s) 10)
+    (apply str (take (count s) (repeat "*")))
+    (str (subs s 0 5)
+         (apply str (take (- (count s) 5) (repeat "*"))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OIDC PROVIDER (GENERIC)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- discover-oidc-config
+  [{:keys [http-client]} {:keys [base-uri] :as opts}]
+  (let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
+        response      (ex/try (http-client {:method :get :uri (str discovery-uri)} {:sync? true}))]
+    (cond
+      (ex/exception? response)
+      (do
+        (l/warn :hint "unable to discover oidc configuration"
+                :discover-uri (str discovery-uri)
+                :cause response)
+        nil)
+
+      (= 200 (:status response))
+      (let [data (json/read (:body response))]
+        {:token-uri (get data :token_endpoint)
+         :auth-uri  (get data :authorization_endpoint)
+         :user-uri  (get data :userinfo_endpoint)})
+
+      :else
+      (do
+        (l/warn :hint "unable to discover OIDC configuration"
+                :uri (str discovery-uri)
+                :response-status-code (:status response))
+        nil))))
+
+(defn- prepare-oidc-opts
+  [cfg]
+  (let [opts {:base-uri      (:base-uri cfg)
+              :client-id     (:client-id cfg)
+              :client-secret (:client-secret cfg)
+              :token-uri     (:token-uri cfg)
+              :auth-uri      (:auth-uri cfg)
+              :user-uri      (:user-uri cfg)
+              :scopes        (:scopes cfg #{"openid" "profile" "email"})
+              :roles-attr    (:roles-attr cfg)
+              :roles         (:roles cfg)
+              :name          "oidc"}
+
+        opts (d/without-nils opts)]
+
+    (when (and (string? (:base-uri opts))
+               (string? (:client-id opts))
+               (string? (:client-secret opts)))
+      (if (and (string? (:token-uri opts))
+               (string? (:user-uri opts))
+               (string? (:auth-uri opts)))
+        opts
+        (some-> (discover-oidc-config cfg opts)
+                (merge opts {:discover? true}))))))
+
+(defmethod ig/prep-key ::generic-provider
+  [_ cfg]
+  (d/without-nils cfg))
+
+(defmethod ig/init-key ::generic-provider
+  [_ cfg]
+  (when (:enabled? cfg)
+    (if-let [opts (prepare-oidc-opts cfg)]
+      (do
+        (l/info :hint "provider initialized"
+                :provider :oidc
+                :method (if (:discover? opts) "discover" "manual")
+                :client-id (:client-id opts)
+                :client-secret (obfuscate-string (:client-secret opts))
+                :scopes (str/join "," (:scopes opts))
+                :auth-uri (:auth-uri opts)
+                :user-uri (:user-uri opts)
+                :token-uri (:token-uri opts)
+                :roles-attr (:roles-attr opts)
+                :roles      (:roles opts))
+        opts)
+      (do
+        (l/warn :hint "unable to initialize auth provider, missing configuration" :provider :oidc)
+        nil))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GOOGLE AUTH PROVIDER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmethod ig/prep-key ::google-provider
+  [_ cfg]
+  (d/without-nils cfg))
+
+(defmethod ig/init-key ::google-provider
+  [_ cfg]
+  (let [opts {:client-id     (:client-id cfg)
+              :client-secret (:client-secret cfg)
+              :scopes        #{"openid" "email" "profile"}
+              :auth-uri      "https://accounts.google.com/o/oauth2/v2/auth"
+              :token-uri     "https://oauth2.googleapis.com/token"
+              :user-uri      "https://openidconnect.googleapis.com/v1/userinfo"
+              :name          "google"}]
+
+    (when (:enabled? cfg)
+      (if (and (string? (:client-id opts))
+               (string? (:client-secret opts)))
+        (do
+          (l/info :hint "provider initialized"
+                  :provider :google
+                  :client-id (:client-id opts)
+                  :client-secret (obfuscate-string (:client-secret opts)))
+          opts)
+
+        (do
+          (l/warn :hint "unable to initialize auth provider, missing configuration" :provider :google)
+          nil)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GITHUB AUTH PROVIDER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- retrieve-github-email
+  [{:keys [http-client]} tdata info]
+  (or (some-> info :email p/resolved)
+      (-> (http-client {:uri "https://api.github.com/user/emails"
+                        :headers {"Authorization" (dm/str (:type tdata) " " (:token tdata))}
+                        :timeout 6000
+                        :method :get})
+          (p/then (fn [{:keys [status body] :as response}]
+                    (when-not (s/int-in-range? 200 300 status)
+                      (ex/raise :type :internal
+                                :code :unable-to-retrieve-github-emails
+                                :hint "unable to retrieve github emails"
+                                :http-status status
+                                :http-body body))
+                    (->> response :body json/read (filter :primary) first :email))))))
+
+(defmethod ig/prep-key ::github-provider
+  [_ cfg]
+  (d/without-nils cfg))
+
+(defmethod ig/init-key ::github-provider
+  [_ cfg]
+  (let [opts {:client-id     (:client-id cfg)
+              :client-secret (:client-secret cfg)
+              :scopes        #{"read:user" "user:email"}
+              :auth-uri      "https://github.com/login/oauth/authorize"
+              :token-uri     "https://github.com/login/oauth/access_token"
+              :user-uri      "https://api.github.com/user"
+              :name          "github"
+
+              ;; Additional hooks for provider specific way of
+              ;; retrieve emails.
+              :get-email-fn           (partial retrieve-github-email cfg)}]
+
+    (when (:enabled? cfg)
+      (if (and (string? (:client-id opts))
+               (string? (:client-secret opts)))
+        (do
+          (l/info :hint "provider initialized"
+                  :provider :github
+                  :client-id (:client-id opts)
+                  :client-secret (obfuscate-string (:client-secret opts)))
+          opts)
+
+        (do
+          (l/warn :hint "unable to initialize auth provider, missing configuration" :provider :github)
+          nil)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GITLAB AUTH PROVIDER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmethod ig/prep-key ::gitlab-provider
+  [_ cfg]
+  (d/without-nils cfg))
+
+(defmethod ig/init-key ::gitlab-provider
+  [_ cfg]
+  (let [base (:base-uri cfg "https://gitlab.com")
+        opts {:base-uri      base
+              :client-id     (:client-id cfg)
+              :client-secret (:client-secret cfg)
+              :scopes        #{"openid" "profile" "email"}
+              :auth-uri      (str base "/oauth/authorize")
+              :token-uri     (str base "/oauth/token")
+              :user-uri      (str base "/oauth/userinfo")
+              :name          "gitlab"}]
+    (when (:enabled? cfg)
+      (if (and (string? (:client-id opts))
+               (string? (:client-secret opts)))
+        (do
+          (l/info :hint "provider initialized"
+                  :provider :gitlab
+                  :base-uri base
+                  :client-id (:client-id opts)
+                  :client-secret (obfuscate-string (:client-secret opts)))
+          opts)
+
+        (do
+          (l/warn :hint "unable to initialize auth provider, missing configuration" :provider :gitlab)
+          nil)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HANDLERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- build-redirect-uri
+  [{:keys [provider] :as cfg}]
+  (let [public (u/uri (:public-uri cfg))]
+    (str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
+
+(defn- build-auth-uri
+  [{:keys [provider] :as cfg} state]
+  (let [params {:client_id (:client-id provider)
+                :redirect_uri (build-redirect-uri cfg)
+                :response_type "code"
+                :state state
+                :scope (str/join " " (:scopes provider []))}
+        query  (u/map->query-string params)]
+    (-> (u/uri (:auth-uri provider))
+        (assoc :query query)
+        (str))))
+
+(defn- qualify-props
+  [provider props]
+  (reduce-kv (fn [result k v]
+               (assoc result (keyword (:name provider) (name k)) v))
+             {}
+             props))
+
+(defn retrieve-access-token
+  [{:keys [provider http-client] :as cfg} code]
+  (let [params {:client_id (:client-id provider)
+                :client_secret (:client-secret provider)
+                :code code
+                :grant_type "authorization_code"
+                :redirect_uri (build-redirect-uri cfg)}
+        req    {:method :post
+                :headers {"content-type" "application/x-www-form-urlencoded"
+                          "accept" "application/json"}
+                :uri (:token-uri provider)
+                :body (u/map->query-string params)}]
+    (p/then
+     (http-client req)
+     (fn [{:keys [status body] :as res}]
+       (if (= status 200)
+         (let [data (json/read body)]
+           {:token (get data :access_token)
+            :type (get data :token_type)})
+         (ex/raise :type :internal
+                   :code :unable-to-retrieve-token
+                   :http-status status
+                   :http-body body))))))
+
+(defn- retrieve-user-info
+  [{:keys [provider http-client] :as cfg} tdata]
+  (letfn [(retrieve []
+            (http-client {:uri (:user-uri provider)
+                          :headers {"Authorization" (str (:type tdata) " " (:token tdata))}
+                          :timeout 6000
+                          :method :get}))
+
+          (validate-response [response]
+            (when-not (s/int-in-range? 200 300 (:status response))
+              (ex/raise :type :internal
+                        :code :unable-to-retrieve-user-info
+                        :hint "unable to retrieve user info"
+                        :http-status (:status response)
+                        :http-body (:body response)))
+            response)
+
+          (get-email [info]
+            ;; Allow providers hook into this for custom email
+            ;; retrieval method.
+            (if-let [get-email-fn (:get-email-fn provider)]
+              (get-email-fn tdata info)
+              (let [attr-kw (cf/get :oidc-email-attr :email)]
+                (get info attr-kw))))
+
+          (get-name [info]
+            (let [attr-kw (cf/get :oidc-name-attr :name)]
+              (get info attr-kw)))
+
+          (process-response [response]
+            (p/let [info  (-> response :body json/read)
+                    email (get-email info)]
+              {:backend  (:name provider)
+               :email    email
+               :fullname (or (get-name info) email)
+               :props    (->> (dissoc info :name :email)
+                              (qualify-props provider))}))
+
+          (validate-info [info]
+            (when-not (s/valid? ::info info)
+              (l/warn :hint "received incomplete profile info object (please set correct scopes)"
+                      :info (pr-str info))
+              (ex/raise :type :internal
+                        :code :incomplete-user-info
+                        :hint "inconmplete user info"
+                        :info info))
+            info)]
+
+    (-> (retrieve)
+        (p/then validate-response)
+        (p/then process-response)
+        (p/then validate-info))))
+
+(s/def ::backend ::us/not-empty-string)
+(s/def ::email ::us/not-empty-string)
+(s/def ::fullname ::us/not-empty-string)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::info
+  (s/keys :req-un [::backend
+                   ::email
+                   ::fullname
+                   ::props]))
+
+(defn retrieve-info
+  [{:keys [tokens provider] :as cfg} {:keys [params] :as request}]
+  (letfn [(validate-oidc [info]
+            ;; If the provider is OIDC, we can proceed to check
+            ;; roles if they are defined.
+            (when (and (= "oidc" (:name provider))
+                       (seq (:roles provider)))
+              (let [provider-roles (into #{} (:roles provider))
+                    profile-roles  (let [attr  (cf/get :oidc-roles-attr :roles)
+                                         roles (get info attr)]
+                                     (cond
+                                       (string? roles) (into #{} (str/words roles))
+                                       (vector? roles) (into #{} roles)
+                                       :else #{}))]
+
+                ;; check if profile has a configured set of roles
+                (when-not (set/subset? provider-roles profile-roles)
+                  (ex/raise :type :internal
+                            :code :unable-to-auth
+                            :hint "not enough permissions"))))
+            info)
+
+          (post-process [state info]
+            (cond-> info
+              (some? (:invitation-token state))
+              (assoc :invitation-token (:invitation-token state))
+
+              ;; If state token comes with props, merge them. The state token
+              ;; props can contain pm_ and utm_ prefixed query params.
+              (map? (:props state))
+              (update :props merge (:props state))))]
+
+    (when-let [error (get params :error)]
+      (ex/raise :type :internal
+                :code :error-on-retrieving-code
+                :error-id error
+                :error-desc (get params :error_description)))
+
+    (let [state  (get params :state)
+          code   (get params :code)
+          state  (tokens :verify {:token state :iss :oauth})]
+      (-> (p/resolved code)
+          (p/then #(retrieve-access-token cfg %))
+          (p/then #(retrieve-user-info cfg %))
+          (p/then' validate-oidc)
+          (p/then' (partial post-process state))))))
+
+(defn- retrieve-profile
+  [{:keys [pool executor] :as cfg} info]
+  (px/with-dispatch executor
+    (with-open [conn (db/open pool)]
+      (some->> (:email info)
+               (profile/retrieve-profile-data-by-email conn)
+               (profile/populate-additional-data conn)
+               (profile/decode-profile-row)))))
+
+(defn- redirect-response
+  [uri]
+  (yrs/response :status 302 :headers {"location" (str uri)}))
+
+(defn- generate-error-redirect
+  [cfg error]
+  (let [uri (-> (u/uri (:public-uri cfg))
+                (assoc :path "/#/auth/login")
+                (assoc :query (u/map->query-string {:error "unable-to-auth" :hint (ex-message error)})))]
+    (redirect-response uri)))
+
+(defn- generate-redirect
+  [{:keys [tokens session audit] :as cfg} request info profile]
+  (if profile
+    (let [sxf    ((:create session) (:id profile))
+          token  (or (:invitation-token info)
+                     (tokens :generate {:iss :auth
+                                        :exp (dt/in-future "15m")
+                                        :profile-id (:id profile)}))
+          params {:token token}
+
+          uri    (-> (u/uri (:public-uri cfg))
+                     (assoc :path "/#/auth/verify-token")
+                     (assoc :query (u/map->query-string params)))]
+
+      (when (fn? audit)
+        (audit :cmd :submit
+               :type "mutation"
+               :name "login"
+               :profile-id (:id profile)
+               :ip-addr (audit/parse-client-ip request)
+               :props (audit/profile->props profile)))
+
+      (->> (redirect-response uri)
+           (sxf request)))
+
+    (let [info   (assoc info
+                        :iss :prepared-register
+                        :is-active true
+                        :exp (dt/in-future {:hours 48}))
+          token  (tokens :generate info)
+          params (d/without-nils
+                  {:token token
+                   :fullname (:fullname info)})
+          uri    (-> (u/uri (:public-uri cfg))
+                     (assoc :path "/#/auth/register/validate")
+                     (assoc :query (u/map->query-string params)))]
+      (redirect-response uri))))
+
+(defn- auth-handler
+  [{:keys [tokens] :as cfg} {:keys [params] :as request}]
+  (let [props (audit/extract-utm-params params)
+        state (tokens :generate
+                      {:iss :oauth
+                       :invitation-token (:invitation-token params)
+                       :props props
+                       :exp (dt/in-future "15m")})
+        uri   (build-auth-uri cfg state)]
+    (yrs/response 200 {:redirect-uri uri})))
+
+(defn- callback-handler
+  [cfg request]
+  (letfn [(process-request []
+            (p/let [info    (retrieve-info cfg request)
+                    profile (retrieve-profile cfg info)]
+              (generate-redirect cfg request info profile)))
+
+          (handle-error [cause]
+            (l/error :hint "error on oauth process" :cause cause)
+            (generate-error-redirect cfg cause))]
+
+    (-> (process-request)
+        (p/catch handle-error))))
+
+(def provider-lookup
+  {:compile
+   (fn [& _]
+     (fn [handler]
+       (fn [{:keys [providers] :as cfg} request]
+         (let [provider (some-> request :path-params :provider keyword)]
+           (if-let [provider (get providers provider)]
+             (handler (assoc cfg :provider provider) request)
+             (ex/raise :type :restriction
+                       :code :provider-not-configured
+                       :provider provider
+                       :hint "provider not configured"))))))})
+
+(s/def ::public-uri ::us/not-empty-string)
+(s/def ::http-client fn?)
+(s/def ::session map?)
+(s/def ::tokens fn?)
+(s/def ::providers map?)
+
+(defmethod ig/pre-init-spec ::routes
+  [_]
+  (s/keys :req-un [::public-uri
+                   ::session
+                   ::tokens
+                   ::http-client
+                   ::providers
+                   ::db/pool
+                   ::wrk/executor]))
+
+(defmethod ig/init-key ::routes
+  [_ {:keys [executor session] :as cfg}]
+  (let [cfg (update cfg :provider d/without-nils)]
+    ["" {:middleware [[(:middleware session)]
+                      [hmw/with-promise-async executor]
+                      [hmw/with-config cfg]
+                      [provider-lookup]
+                      ]}
+     ;; We maintain the both URI prefixes for backward compatibility.
+
+     ["/auth/oauth"
+      ["/:provider"
+       {:handler auth-handler
+        :allowed-methods #{:post}}]
+      ["/:provider/callback"
+       {:handler callback-handler
+        :allowed-methods #{:get}}]]
+
+     ["/auth/oidc"
+      ["/:provider"
+       {:handler auth-handler
+        :allowed-methods #{:post}}]
+      ["/:provider/callback"
+       {:handler callback-handler
+        :allowed-methods #{:get}}]]]))

backend/src/app/cli/fixtures.clj

@@ -1,258 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.cli.fixtures
-  "A initial fixtures."
-  (:require
-   [app.common.pages :as cp]
-   [app.common.uuid :as uuid]
-   [app.db :as db]
-   [app.main :as main]
-   [app.rpc.mutations.profile :as profile]
-   [app.util.blob :as blob]
-   [app.util.logging :as l]
-   [buddy.hashers :as hashers]
-   [integrant.core :as ig]))
-
-(defn- mk-uuid
-  [prefix & args]
-  (uuid/namespaced uuid/zero (apply str prefix (interpose "-" args))))
-
-;; --- Profiles creation
-
-(def password (hashers/derive "123123"))
-
-(def preset-small
-  {:num-teams 5
-   :num-profiles 5
-   :num-profiles-per-team 5
-   :num-projects-per-team 5
-   :num-files-per-project 5
-   :num-draft-files-per-profile 10})
-
-(defn- rng-ids
-  [rng n max]
-  (let [stream (->> (.longs rng 0 max)
-                    (.iterator)
-                    (iterator-seq))]
-    (reduce (fn [acc item]
-              (if (= (count acc) n)
-                (reduced acc)
-                (conj acc item)))
-            #{}
-            stream)))
-
-(defn- rng-vec
-  [rng vdata n]
-  (let [ids (rng-ids rng n (count vdata))]
-    (mapv #(nth vdata %) ids)))
-
-(defn- rng-nth
-  [rng vdata]
-  (let [stream (->> (.longs rng 0 (count vdata))
-                    (.iterator)
-                    (iterator-seq))]
-    (nth vdata (first stream))))
-
-(defn- collect
-  [f items]
-  (reduce #(conj %1 (f %2)) [] items))
-
-(defn- register-profile
-  [conn params]
-  (->> (#'profile/create-profile conn params)
-       (#'profile/create-profile-relations conn)))
-
-(defn impl-run
-  [pool opts]
-  (let [rng (java.util.Random. 1)]
-    (letfn [(create-profile [conn index]
-              (let [id   (mk-uuid "profile" index)
-                    _    (l/info :action "create profile"
-                                 :index index
-                                 :id id)
-
-                    prof (register-profile conn
-                                           {:id id
-                                            :fullname (str "Profile " index)
-                                            :password "123123"
-                                            :is-demo true
-                                            :email (str "profile" index "@example.com")})
-                    team-id  (:default-team-id prof)
-                    owner-id id]
-                (let [project-ids (collect (partial create-project conn team-id owner-id)
-                                           (range (:num-projects-per-team opts)))]
-                  (run! (partial create-files conn owner-id) project-ids))
-                prof))
-
-            (create-profiles [conn]
-              (l/info :action "create profiles")
-              (collect (partial create-profile conn)
-                       (range (:num-profiles opts))))
-
-            (create-team [conn index]
-              (let [id (mk-uuid "team" index)
-                    name (str "Team" index)]
-                (l/info :action "create team"
-                        :index index
-                        :id id)
-                (db/insert! conn :team {:id id
-                                        :name name})
-                id))
-
-            (create-teams [conn]
-              (l/info :action "create teams")
-              (collect (partial create-team conn)
-                       (range (:num-teams opts))))
-
-            (create-file [conn owner-id project-id index]
-              (let [id (mk-uuid "file" project-id index)
-                    name (str "file" index)
-                    data (cp/make-file-data id)]
-                (l/info :action "create file"
-                        :index index
-                        :id id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-files [conn owner-id project-id]
-              (l/info :action "create files")
-              (run! (partial create-file conn owner-id project-id)
-                    (range (:num-files-per-project opts))))
-
-            (create-project [conn team-id owner-id index]
-              (let [id        (if index
-                                (mk-uuid "project" team-id index)
-                                (mk-uuid "project" team-id))
-                    name      (if index
-                                (str "project " index)
-                                "Drafts")
-                    is-default (nil? index)]
-                (l/info :action "create project"
-                        :index index
-                        :id id)
-                (db/insert! conn :project
-                            {:id id
-                             :team-id team-id
-                             :is-default is-default
-                             :name name})
-                (db/insert! conn :project-profile-rel
-                            {:project-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-projects [conn team-id profile-ids]
-              (l/info :action "create projects")
-              (let [owner-id (rng-nth rng profile-ids)
-                    project-ids (conj
-                                  (collect (partial create-project conn team-id owner-id)
-                                         (range (:num-projects-per-team opts)))
-                                  (create-project conn team-id owner-id nil))]
-                (run! (partial create-files conn owner-id) project-ids)))
-
-            (assign-profile-to-team [conn team-id owner? profile-id]
-              (db/insert! conn :team-profile-rel
-                          {:team-id team-id
-                           :profile-id profile-id
-                           :is-owner owner?
-                           :is-admin true
-                           :can-edit true}))
-
-            (setup-team [conn team-id profile-ids]
-              (l/info :action "setup team"
-                      :team-id team-id
-                      :profile-ids (pr-str profile-ids))
-              (assign-profile-to-team conn team-id true (first profile-ids))
-              (run! (partial assign-profile-to-team conn team-id false)
-                    (rest profile-ids))
-              (create-projects conn team-id profile-ids))
-
-            (assign-teams-and-profiles [conn teams profiles]
-              (l/info :action "assign teams and profiles")
-              (loop [team-id (first teams)
-                     teams (rest teams)]
-                (when-not (nil? team-id)
-                  (let [n-profiles-team (:num-profiles-per-team opts)
-                        selected-profiles (rng-vec rng profiles n-profiles-team)]
-                    (setup-team conn team-id selected-profiles)
-                    (recur (first teams)
-                           (rest teams))))))
-
-            (create-draft-file [conn owner index]
-              (let [owner-id   (:id owner)
-                    id         (mk-uuid "file" "draft" owner-id index)
-                    name       (str "file" index)
-                    project-id (:default-project-id owner)
-                    data       (cp/make-file-data id)]
-
-                (l/info :action "create draft file"
-                        :index index
-                        :id id)
-                (db/insert! conn :file
-                            {:id id
-                             :data (blob/encode data)
-                             :project-id project-id
-                             :name name})
-                (db/insert! conn :file-profile-rel
-                            {:file-id id
-                             :profile-id owner-id
-                             :is-owner true
-                             :is-admin true
-                             :can-edit true})
-                id))
-
-            (create-draft-files [conn profile]
-              (run! (partial create-draft-file conn profile)
-                    (range (:num-draft-files-per-profile opts))))
-            ]
-      (db/with-atomic [conn pool]
-        (let [profiles (create-profiles conn)
-              teams    (create-teams conn)]
-          (assign-teams-and-profiles conn teams (map :id profiles))
-          (run! (partial create-draft-files conn) profiles))))))
-
-(defn run-in-system
-  [system preset]
-  (let [pool   (:app.db/pool system)
-        preset (if (map? preset)
-                 preset
-                 (case preset
-                   (nil "small" :small) preset-small
-                   ;; "medium" preset-medium
-                   ;; "big" preset-big
-                   preset-small))]
-    (impl-run pool preset)))
-
-(defn run
-  [{:keys [preset] :or {preset :small}}]
-  (let [config (select-keys main/system-config
-                            [:app.db/pool
-                             :app.telemetry/migrations
-                             :app.migrations/migrations
-                             :app.migrations/all
-                             :app.metrics/metrics])
-        _      (ig/load-namespaces config)
-        system (-> (ig/prep config)
-                   (ig/init))]
-    (try
-      (run-in-system system preset)
-      (catch Exception e
-        (l/error :hint "unhandled exception" :cause e))
-      (finally
-        (ig/halt! system)))))

backend/src/app/cli/manage.clj

@@ -7,11 +7,12 @@
 (ns app.cli.manage
   "A manage cli api."
   (:require
+   [app.common.logging :as l]
    [app.db :as db]
    [app.main :as main]
+   [app.rpc.commands.auth :as cmd.auth]
    [app.rpc.mutations.profile :as profile]
    [app.rpc.queries.profile :refer [retrieve-profile-data-by-email]]
-   [app.util.logging :as l]
    [clojure.string :as str]
    [clojure.tools.cli :refer [parse-opts]]
    [integrant.core :as ig])
@@ -54,13 +55,13 @@
                                          :type :password}))]
     (try
       (db/with-atomic [conn (:app.db/pool system)]
-        (->> (profile/create-profile conn
+        (->> (cmd.auth/create-profile conn
                                      {:fullname fullname
                                       :email email
                                       :password password
                                       :is-active true
                                       :is-demo false})
-             (profile/create-profile-relations conn)))
+             (cmd.auth/create-profile-relations conn)))
 
       (when (pos? (:verbosity options))
         (println "User created successfully."))
@@ -140,7 +141,6 @@
   indicating the action the program should take and the options provided."
   [args]
   (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)]
-    ;; (pp/pprint opts)
     (cond
       (:help options) ; help => exit OK with usage summary
       {:exit-message (usage summary) :ok? true}

backend/src/app/cli/migrate_media.clj

@@ -1,129 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.cli.migrate-media
-  (:require
-   [app.common.media :as cm]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.main :as main]
-   [app.storage :as sto]
-   [app.util.logging :as l]
-   [cuerdas.core :as str]
-   [datoteka.core :as fs]
-   [integrant.core :as ig]))
-
-(declare migrate-profiles)
-(declare migrate-teams)
-(declare migrate-file-media)
-
-(defn run-in-system
-  [system]
-  (db/with-atomic [conn (:app.db/pool system)]
-    (let [system (assoc system ::conn conn)]
-      (migrate-profiles system)
-      (migrate-teams system)
-      (migrate-file-media system))
-    system))
-
-(defn run
-  []
-  (let [config (select-keys main/system-config
-                            [:app.db/pool
-                             :app.migrations/migrations
-                             :app.metrics/metrics
-                             :app.storage.s3/backend
-                             :app.storage.db/backend
-                             :app.storage.fs/backend
-                             :app.storage/storage])]
-    (ig/load-namespaces config)
-    (try
-      (-> (ig/prep config)
-          (ig/init)
-          (run-in-system)
-          (ig/halt!))
-      (catch Exception e
-        (l/error :hint "unhandled exception" :cause e)))))
-
-
-;; --- IMPL
-
-(defn migrate-profiles
-  [{:keys [::conn] :as system}]
-  (letfn [(retrieve-profiles [conn]
-            (->> (db/exec! conn ["select * from profile"])
-                 (filter #(not (str/empty? (:photo %))))
-                 (seq)))]
-    (let [base    (fs/path (cf/get :storage-fs-old-directory))
-          storage (-> (:app.storage/storage system)
-                      (assoc :conn conn))]
-      (doseq [profile (retrieve-profiles conn)]
-        (let [path  (fs/path (:photo profile))
-              full  (-> (fs/join base path)
-                        (fs/normalize))
-              ext   (fs/ext path)
-              mtype (cm/format->mtype (keyword ext))
-              obj   (sto/put-object storage {:content (sto/content full)
-                                             :content-type mtype})]
-          (db/update! conn :profile
-                      {:photo-id (:id obj)}
-                      {:id (:id profile)}))))))
-
-(defn migrate-teams
-  [{:keys [::conn] :as system}]
-  (letfn [(retrieve-teams [conn]
-            (->> (db/exec! conn ["select * from team"])
-                 (filter #(not (str/empty? (:photo %))))
-                 (seq)))]
-    (let [base    (fs/path (cf/get :storage-fs-old-directory))
-          storage (-> (:app.storage/storage system)
-                      (assoc :conn conn))]
-      (doseq [team (retrieve-teams conn)]
-        (let [path  (fs/path (:photo team))
-              full  (-> (fs/join base path)
-                        (fs/normalize))
-              ext   (fs/ext path)
-              mtype (cm/format->mtype (keyword ext))
-              obj   (sto/put-object storage {:content (sto/content full)
-                                             :content-type mtype})]
-          (db/update! conn :team
-                      {:photo-id (:id obj)}
-                      {:id (:id team)}))))))
-
-
-
-(defn migrate-file-media
-  [{:keys [::conn] :as system}]
-  (letfn [(retrieve-media-objects [conn]
-            (->> (db/exec! conn ["select fmo.id, fmo.path, fth.path as thumbnail_path
-                                    from file_media_object as fmo
-                                    join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"])
-                 (seq)))]
-    (let [base    (fs/path (cf/get :storage-fs-old-directory))
-          storage (-> (:app.storage/storage system)
-                      (assoc :conn conn))]
-      (doseq [mobj (retrieve-media-objects conn)]
-        (let [img-path  (fs/path (:path mobj))
-              thm-path  (fs/path (:thumbnail-path mobj))
-              img-path  (-> (fs/join base img-path)
-                            (fs/normalize))
-              thm-path  (-> (fs/join base thm-path)
-                            (fs/normalize))
-              img-ext   (fs/ext img-path)
-              thm-ext   (fs/ext thm-path)
-
-              img-mtype (cm/format->mtype (keyword img-ext))
-              thm-mtype (cm/format->mtype (keyword thm-ext))
-
-              img-obj   (sto/put-object storage {:content (sto/content img-path)
-                                                 :content-type img-mtype})
-              thm-obj   (sto/put-object storage {:content (sto/content thm-path)
-                                                 :content-type thm-mtype})]
-
-          (db/update! conn :file-media-object
-                      {:media-id (:id img-obj)
-                       :thumbnail-id (:id thm-obj)}
-                      {:id (:id mobj)}))))))

backend/src/app/config.clj

@@ -10,6 +10,8 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.flags :as flags]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.version :as v]
    [app.util.time :as dt]
@@ -40,36 +42,28 @@
     data))
 
 (def defaults
-  {:http-server-port 6060
-   :host "devenv"
-   :tenant "dev"
-   :database-uri "postgresql://postgres/penpot"
+  {:database-uri "postgresql://postgres/penpot"
    :database-username "penpot"
    :database-password "penpot"
 
-   :default-blob-version 3
+   :default-blob-version 4
    :loggers-zmq-uri "tcp://localhost:45556"
 
-   :asserts-enabled false
+   :file-change-snapshot-every 5
+   :file-change-snapshot-timeout "3h"
 
    :public-uri "http://localhost:3449"
-   :redis-uri "redis://redis/0"
+   :host "localhost"
+   :tenant "main"
 
+   :redis-uri "redis://redis/0"
    :srepl-host "127.0.0.1"
    :srepl-port 6062
 
    :assets-storage-backend :assets-fs
    :storage-assets-fs-directory "assets"
 
-   :feedback-destination "info@example.com"
-   :feedback-enabled false
-
    :assets-path "/internal/assets/"
-
-   :rlimits-password 10
-   :rlimits-image 2
-
-   :smtp-enabled false
    :smtp-default-reply-to "Penpot <no-reply@example.com>"
    :smtp-default-from "Penpot <no-reply@example.com>"
 
@@ -79,40 +73,51 @@
    :profile-bounce-max-age (dt/duration {:days 7})
    :profile-bounce-threshold 10
 
-   :allow-demo-users true
-   :registration-enabled true
-
-   :telemetry-enabled false
    :telemetry-uri "https://telemetry.penpot.app/"
 
    :ldap-user-query "(|(uid=:username)(mail=:username))"
    :ldap-attrs-username "uid"
    :ldap-attrs-email "mail"
    :ldap-attrs-fullname "cn"
-   :ldap-attrs-photo "jpegPhoto"
 
    ;; a server prop key where initial project is stored.
-   :initial-project-skey "initial-project"
-   })
+   :initial-project-skey "initial-project"})
 
-(s/def ::audit-enabled ::us/boolean)
-(s/def ::audit-archive-enabled ::us/boolean)
-(s/def ::audit-archive-uri ::us/string)
-(s/def ::audit-archive-gc-enabled ::us/boolean)
-(s/def ::audit-archive-gc-max-age ::dt/duration)
+
+(s/def ::media-max-file-size ::us/integer)
+
+(s/def ::flags ::us/vec-of-valid-keywords)
+(s/def ::telemetry-enabled ::us/boolean)
+
+(s/def ::audit-log-archive-uri ::us/string)
+(s/def ::audit-log-gc-max-age ::dt/duration)
+
+(s/def ::admins ::us/set-of-non-empty-strings)
+(s/def ::file-change-snapshot-every ::us/integer)
+(s/def ::file-change-snapshot-timeout ::dt/duration)
+
+(s/def ::default-executor-parallelism ::us/integer)
+(s/def ::blocking-executor-parallelism ::us/integer)
+(s/def ::worker-executor-parallelism ::us/integer)
+
+(s/def ::authenticated-cookie-domain ::us/string)
+(s/def ::authenticated-cookie-name ::us/string)
+(s/def ::auth-token-cookie-name ::us/string)
+(s/def ::auth-token-cookie-max-age ::dt/duration)
 
 (s/def ::secret-key ::us/string)
 (s/def ::allow-demo-users ::us/boolean)
-(s/def ::asserts-enabled ::us/boolean)
 (s/def ::assets-path ::us/string)
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
 (s/def ::database-username (s/nilable ::us/string))
+(s/def ::database-readonly ::us/boolean)
+(s/def ::database-min-pool-size ::us/integer)
+(s/def ::database-max-pool-size ::us/integer)
+
 (s/def ::default-blob-version ::us/integer)
 (s/def ::error-report-webhook ::us/string)
-(s/def ::feedback-destination ::us/string)
-(s/def ::feedback-enabled ::us/boolean)
-(s/def ::feedback-token ::us/string)
+(s/def ::user-feedback-destination ::us/string)
 (s/def ::github-client-id ::us/string)
 (s/def ::github-client-secret ::us/string)
 (s/def ::gitlab-base-uri ::us/string)
@@ -126,18 +131,21 @@
 (s/def ::oidc-token-uri ::us/string)
 (s/def ::oidc-auth-uri ::us/string)
 (s/def ::oidc-user-uri ::us/string)
-(s/def ::oidc-scopes ::us/set-of-str)
-(s/def ::oidc-roles ::us/set-of-str)
+(s/def ::oidc-scopes ::us/set-of-non-empty-strings)
+(s/def ::oidc-roles ::us/set-of-non-empty-strings)
 (s/def ::oidc-roles-attr ::us/keyword)
+(s/def ::oidc-email-attr ::us/keyword)
+(s/def ::oidc-name-attr ::us/keyword)
 (s/def ::host ::us/string)
 (s/def ::http-server-port ::us/integer)
-(s/def ::http-session-idle-max-age ::dt/duration)
-(s/def ::http-session-updater-batch-max-age ::dt/duration)
-(s/def ::http-session-updater-batch-max-size ::us/integer)
+(s/def ::http-server-host ::us/string)
+(s/def ::http-server-max-body-size ::us/integer)
+(s/def ::http-server-max-multipart-body-size ::us/integer)
+(s/def ::http-server-io-threads ::us/integer)
+(s/def ::http-server-worker-threads ::us/integer)
 (s/def ::initial-project-skey ::us/string)
 (s/def ::ldap-attrs-email ::us/string)
 (s/def ::ldap-attrs-fullname ::us/string)
-(s/def ::ldap-attrs-photo ::us/string)
 (s/def ::ldap-attrs-username ::us/string)
 (s/def ::ldap-base-dn ::us/string)
 (s/def ::ldap-bind-dn ::us/string)
@@ -157,13 +165,13 @@
 (s/def ::profile-complaint-threshold ::us/integer)
 (s/def ::public-uri ::us/string)
 (s/def ::redis-uri ::us/string)
-(s/def ::registration-domain-whitelist ::us/set-of-str)
-(s/def ::registration-enabled ::us/boolean)
-(s/def ::rlimits-image ::us/integer)
-(s/def ::rlimits-password ::us/integer)
+(s/def ::registration-domain-whitelist ::us/set-of-non-empty-strings)
+(s/def ::rlimit-font ::us/integer)
+(s/def ::rlimit-file-update ::us/integer)
+(s/def ::rlimit-image ::us/integer)
+(s/def ::rlimit-password ::us/integer)
 (s/def ::smtp-default-from ::us/string)
 (s/def ::smtp-default-reply-to ::us/string)
-(s/def ::smtp-enabled ::us/boolean)
 (s/def ::smtp-host ::us/string)
 (s/def ::smtp-password (s/nilable ::us/string))
 (s/def ::smtp-port ::us/integer)
@@ -177,31 +185,45 @@
 (s/def ::storage-assets-fs-directory ::us/string)
 (s/def ::storage-assets-s3-bucket ::us/string)
 (s/def ::storage-assets-s3-region ::us/keyword)
+(s/def ::storage-assets-s3-endpoint ::us/string)
 (s/def ::storage-fdata-s3-bucket ::us/string)
 (s/def ::storage-fdata-s3-region ::us/keyword)
 (s/def ::storage-fdata-s3-prefix ::us/string)
-(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::storage-fdata-s3-endpoint ::us/string)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
 
+(s/def ::sentry-trace-sample-rate ::us/number)
+(s/def ::sentry-attach-stack-trace ::us/boolean)
+(s/def ::sentry-debug ::us/boolean)
+(s/def ::sentry-dsn ::us/string)
+
 (s/def ::config
   (s/keys :opt-un [::secret-key
+                   ::flags
+                   ::admins
                    ::allow-demo-users
-                   ::audit-enabled
-                   ::audit-archive-enabled
-                   ::audit-archive-uri
-                   ::audit-archive-gc-enabled
-                   ::audit-archive-gc-max-age
-                   ::asserts-enabled
+                   ::audit-log-archive-uri
+                   ::audit-log-gc-max-age
+                   ::auth-token-cookie-name
+                   ::auth-token-cookie-max-age
+                   ::authenticated-cookie-name
+                   ::authenticated-cookie-domain
                    ::database-password
                    ::database-uri
                    ::database-username
+                   ::database-readonly
+                   ::database-min-pool-size
+                   ::database-max-pool-size
                    ::default-blob-version
                    ::error-report-webhook
-                   ::feedback-destination
-                   ::feedback-enabled
-                   ::feedback-token
+                   ::default-executor-parallelism
+                   ::blocking-executor-parallelism
+                   ::worker-executor-parallelism
+                   ::file-change-snapshot-every
+                   ::file-change-snapshot-timeout
+                   ::user-feedback-destination
                    ::github-client-id
                    ::github-client-secret
                    ::gitlab-base-uri
@@ -217,16 +239,19 @@
                    ::oidc-user-uri
                    ::oidc-scopes
                    ::oidc-roles-attr
+                   ::oidc-email-attr
+                   ::oidc-name-attr
                    ::oidc-roles
                    ::host
+                   ::http-server-host
                    ::http-server-port
-                   ::http-session-idle-max-age
-                   ::http-session-updater-batch-max-age
-                   ::http-session-updater-batch-max-size
+                   ::http-server-max-body-size
+                   ::http-server-max-multipart-body-size
+                   ::http-server-io-threads
+                   ::http-server-worker-threads
                    ::initial-project-skey
                    ::ldap-attrs-email
                    ::ldap-attrs-fullname
-                   ::ldap-attrs-photo
                    ::ldap-attrs-username
                    ::ldap-base-dn
                    ::ldap-bind-dn
@@ -239,6 +264,7 @@
                    ::local-assets-uri
                    ::loggers-loki-uri
                    ::loggers-zmq-uri
+                   ::media-max-file-size
                    ::profile-bounce-max-age
                    ::profile-bounce-threshold
                    ::profile-complaint-max-age
@@ -246,38 +272,51 @@
                    ::public-uri
                    ::redis-uri
                    ::registration-domain-whitelist
-                   ::registration-enabled
-                   ::rlimits-image
-                   ::rlimits-password
+                   ::rlimit-font
+                   ::rlimit-file-update
+                   ::rlimit-image
+                   ::rlimit-password
+                   ::sentry-dsn
+                   ::sentry-debug
+                   ::sentry-attach-stack-trace
+                   ::sentry-trace-sample-rate
                    ::smtp-default-from
                    ::smtp-default-reply-to
-                   ::smtp-enabled
                    ::smtp-host
                    ::smtp-password
                    ::smtp-port
                    ::smtp-ssl
                    ::smtp-tls
                    ::smtp-username
-
                    ::srepl-host
                    ::srepl-port
-
                    ::assets-storage-backend
                    ::storage-assets-fs-directory
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
-
+                   ::storage-assets-s3-endpoint
                    ::fdata-storage-backend
                    ::storage-fdata-s3-bucket
                    ::storage-fdata-s3-region
                    ::storage-fdata-s3-prefix
-
+                   ::storage-fdata-s3-endpoint
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer
                    ::telemetry-with-taiga
                    ::tenant]))
 
+(def default-flags
+  [:enable-backend-api-doc
+   :enable-backend-worker
+   :enable-secure-session-cookies])
+
+(defn- parse-flags
+  [config]
+  (flags/parse flags/default
+               default-flags
+               (:flags config)))
+
 (defn read-env
   [prefix]
   (let [prefix (str prefix "-")
@@ -300,15 +339,22 @@
       (when (ex/ex-info? e)
         (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")
         (println "Error on validating configuration:")
-        (println (:explain (ex-data e))
-        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
+        (println (us/pretty-explain (ex-data e)))
+        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"))
       (throw e))))
 
-(def version (v/parse (or (some-> (io/resource "version.txt")
-                                  (slurp)
-                                  (str/trim))
-                          "%version%")))
-(def config (atom (read-config)))
+(def version
+  (v/parse (or (some-> (io/resource "version.txt")
+                       (slurp)
+                       (str/trim))
+               "%version%")))
+
+(defonce ^:dynamic config (read-config))
+
+(defonce ^:dynamic flags
+  (let [flags (parse-flags config)]
+    (l/info :hint "flags initialized" :flags (str/join "," (map name flags)))
+    flags))
 
 (def deletion-delay
   (dt/duration {:days 7}))
@@ -316,9 +362,9 @@
 (defn get
   "A configuration getter. Helps code be more testable."
   ([key]
-   (c/get @config key))
+   (c/get config key))
   ([key default]
-   (c/get @config key default)))
+   (c/get config key default)))
 
 ;; Set value for all new threads bindings.
-(alter-var-root #'*assert* (constantly (get :asserts-enabled)))
+(alter-var-root #'*assert* (constantly (contains? flags :backend-asserts)))

backend/src/app/db.clj

@@ -9,13 +9,13 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.db.sql :as sql]
    [app.metrics :as mtx]
    [app.util.json :as json]
-   [app.util.logging :as l]
    [app.util.migrations :as mg]
    [app.util.time :as dt]
    [clojure.java.io :as io]
@@ -27,14 +27,16 @@
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
+   java.io.InputStream
+   java.io.OutputStream
    java.lang.AutoCloseable
    java.sql.Connection
    java.sql.Savepoint
    org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
+   org.postgresql.jdbc.PgArray
    org.postgresql.largeobject.LargeObject
    org.postgresql.largeobject.LargeObjectManager
-   org.postgresql.jdbc.PgArray
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
@@ -45,76 +47,116 @@
 ;; Initialization
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare instrument-jdbc!)
+(declare apply-migrations!)
 
-(s/def ::name keyword?)
-(s/def ::uri ::us/not-empty-string)
-(s/def ::min-pool-size ::us/integer)
-(s/def ::max-pool-size ::us/integer)
+(s/def ::connection-timeout ::us/integer)
+(s/def ::max-size ::us/integer)
+(s/def ::min-size ::us/integer)
 (s/def ::migrations map?)
+(s/def ::name keyword?)
+(s/def ::password ::us/string)
+(s/def ::uri ::us/not-empty-string)
+(s/def ::username ::us/string)
+(s/def ::validation-timeout ::us/integer)
+(s/def ::read-only? ::us/boolean)
 
-(defmethod ig/pre-init-spec ::pool [_]
-  (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size ::migrations ::mtx/metrics]))
+(s/def ::pool-options
+  (s/keys :opt-un [::uri ::name
+                   ::min-size
+                   ::max-size
+                   ::connection-timeout
+                   ::validation-timeout
+                   ::migrations
+                   ::username
+                   ::password
+                   ::mtx/metrics
+                   ::read-only?]))
+
+(def defaults
+  {:name :main
+   :min-size 0
+   :max-size 30
+   :connection-timeout 10000
+   :validation-timeout 10000
+   :idle-timeout 120000 ; 2min
+   :max-lifetime 1800000 ; 30m
+   :read-only? false})
+
+(defmethod ig/prep-key ::pool
+  [_ cfg]
+  (merge defaults (d/without-nils cfg)))
+
+;; Don't validate here, just validate that a map is received.
+(defmethod ig/pre-init-spec ::pool [_] ::pool-options)
 
 (defmethod ig/init-key ::pool
-  [_ {:keys [migrations metrics] :as cfg}]
-  (l/info :action "initialize connection pool"
-          :name (d/name (:name cfg))
-          :uri (:uri cfg))
-  (instrument-jdbc! (:registry metrics))
-  (let [pool (create-pool cfg)]
-    (when (seq migrations)
-      (with-open [conn ^AutoCloseable (open pool)]
-        (mg/setup! conn)
-        (doseq [[name steps] migrations]
-          (mg/migrate! conn {:name (d/name name) :steps steps}))))
-    pool))
+  [_ {:keys [migrations read-only? uri] :as cfg}]
+  (if uri
+    (let [pool (create-pool cfg)]
+      (l/info :hint "initialize connection pool"
+              :name (d/name (:name cfg))
+              :uri uri
+              :read-only read-only?
+              :with-credentials (and (contains? cfg :username)
+                                     (contains? cfg :password))
+              :min-size (:min-size cfg)
+              :max-size (:max-size cfg))
+      (when-not read-only?
+        (some->> (seq migrations) (apply-migrations! pool)))
+      pool)
+
+    (do
+      (l/warn :hint "unable to initialize pool, missing url"
+              :name (d/name (:name cfg))
+              :read-only read-only?)
+      nil)))
 
 (defmethod ig/halt-key! ::pool
   [_ pool]
-  (.close ^HikariDataSource pool))
+  (when pool
+    (.close ^HikariDataSource pool)))
 
-(defn- instrument-jdbc!
-  [registry]
-  (mtx/instrument-vars!
-   [#'next.jdbc/execute-one!
-    #'next.jdbc/execute!]
-   {:registry registry
-    :type :counter
-    :name "database_query_total"
-    :help "An absolute counter of database queries."}))
+(defn- apply-migrations!
+  [pool migrations]
+  (with-open [conn ^AutoCloseable (open pool)]
+    (mg/setup! conn)
+    (doseq [[name steps] migrations]
+      (mg/migrate! conn {:name (d/name name) :steps steps}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; API & Impl
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (def initsql
-  (str "SET statement_timeout = 120000;\n"
-       "SET idle_in_transaction_session_timeout = 120000;"))
+  (str "SET statement_timeout = 300000;\n"
+       "SET idle_in_transaction_session_timeout = 300000;"))
 
 (defn- create-datasource-config
-  [{:keys [metrics] :as cfg}]
-  (let [dburi    (:uri cfg)
-        username (:username cfg)
-        password (:password cfg)
-        config   (HikariConfig.)
-        mtf      (PrometheusMetricsTrackerFactory. (:registry metrics))]
+  [{:keys [metrics uri] :as cfg}]
+  (let [config (HikariConfig.)]
     (doto config
-      (.setJdbcUrl (str "jdbc:" dburi))
-      (.setPoolName (d/name (:name cfg)))
+      (.setJdbcUrl           (str "jdbc:" uri))
+      (.setPoolName          (d/name (:name cfg)))
       (.setAutoCommit true)
-      (.setReadOnly false)
-      (.setConnectionTimeout 8000)  ;; 8seg
-      (.setValidationTimeout 8000)  ;; 8seg
-      (.setIdleTimeout 120000)      ;; 2min
-      (.setMaxLifetime 1800000)     ;; 30min
-      (.setMinimumIdle (:min-pool-size cfg 0))
-      (.setMaximumPoolSize (:max-pool-size cfg 30))
-      (.setMetricsTrackerFactory mtf)
+      (.setReadOnly          (:read-only? cfg))
+      (.setConnectionTimeout (:connection-timeout cfg))
+      (.setValidationTimeout (:validation-timeout cfg))
+      (.setIdleTimeout       (:idle-timeout cfg))
+      (.setMaxLifetime       (:max-lifetime cfg))
+      (.setMinimumIdle       (:min-size cfg))
+      (.setMaximumPoolSize   (:max-size cfg))
       (.setConnectionInitSql initsql)
       (.setInitializationFailTimeout -1))
-    (when username (.setUsername config username))
-    (when password (.setPassword config password))
+
+    ;; When metrics namespace is provided
+    (when metrics
+      (->> (:registry metrics)
+           (PrometheusMetricsTrackerFactory.)
+           (.setMetricsTrackerFactory config)))
+
+    (some->> ^String (:username cfg) (.setUsername config))
+    (some->> ^String (:password cfg) (.setPassword config))
+
     config))
 
 (defn pool?
@@ -123,11 +165,15 @@
 
 (s/def ::pool pool?)
 
-(defn pool-closed?
+(defn closed?
   [pool]
   (.isClosed ^HikariDataSource pool))
 
-(defn- create-pool
+(defn read-only?
+  [pool]
+  (.isReadOnly ^HikariDataSource pool))
+
+(defn create-pool
   [cfg]
   (let [dsc (create-datasource-config cfg)]
     (jdbc-dt/read-as-instant)
@@ -179,7 +225,7 @@
   [& args]
   `(jdbc/with-transaction ~@args))
 
-(defn ^Connection open
+(defn open
   [pool]
   (jdbc/get-connection pool))
 
@@ -199,21 +245,21 @@
   ([ds table params opts]
    (exec-one! ds
               (sql/insert table params opts)
-              (assoc opts :return-keys true))))
+              (merge {:return-keys true} opts))))
 
 (defn insert-multi!
   ([ds table cols rows] (insert-multi! ds table cols rows nil))
   ([ds table cols rows opts]
    (exec! ds
           (sql/insert-multi table cols rows opts)
-          (assoc opts :return-keys true))))
+          (merge {:return-keys true} opts))))
 
 (defn update!
   ([ds table params where] (update! ds table params where nil))
   ([ds table params where opts]
    (exec-one! ds
               (sql/update table params where opts)
-              (assoc opts :return-keys true))))
+              (merge {:return-keys true} opts))))
 
 (defn delete!
   ([ds table params] (delete! ds table params nil))
@@ -231,9 +277,9 @@
 (defn get-by-params
   ([ds table params]
    (get-by-params ds table params nil))
-  ([ds table params {:keys [uncheked] :or {uncheked false} :as opts}]
+  ([ds table params {:keys [check-not-found] :or {check-not-found true} :as opts}]
    (let [res (exec-one! ds (sql/select table params opts))]
-     (when (and (not uncheked) (or (not res) (is-deleted? res)))
+     (when (and check-not-found (or (not res) (is-deleted? res)))
        (ex/raise :type :not-found
                  :table table
                  :hint "database object not found"))
@@ -267,13 +313,28 @@
   (instance? PGpoint v))
 
 (defn pgarray?
-  [v]
-  (instance? PgArray v))
+  ([v] (instance? PgArray v))
+  ([v type]
+   (and (instance? PgArray v)
+        (= type (.getBaseTypeName ^PgArray v)))))
 
 (defn pgarray-of-uuid?
   [v]
   (and (pgarray? v) (= "uuid" (.getBaseTypeName ^PgArray v))))
 
+(defn decode-pgarray
+  ([v] (some->> ^PgArray v .getArray vec))
+  ([v in] (some->> ^PgArray v .getArray (into in)))
+  ([v in xf]  (some->> ^PgArray v .getArray (into in xf))))
+
+(defn pgarray->set
+  [v]
+  (set (.getArray ^PgArray v)))
+
+(defn pgarray->vector
+  [v]
+  (vec (.getArray ^PgArray v)))
+
 (defn pgpoint
   [p]
   (PGpoint. (:x p) (:y p)))
@@ -285,7 +346,6 @@
       (.createArrayOf conn ^String type (into-array Object objects))
       (.createArrayOf conn ^String type objects))))
 
-
 (defn decode-pgpoint
   [^PGpoint v]
   (gpt/point (.-x v) (.-y v)))
@@ -331,7 +391,7 @@
         val (.getValue o)]
     (if (or (= typ "json")
             (= typ "jsonb"))
-      (json/decode-str val)
+      (json/read val)
       val)))
 
 (defn decode-transit-pgobject
@@ -367,16 +427,7 @@
   [data]
   (doto (org.postgresql.util.PGobject.)
     (.setType "jsonb")
-    (.setValue (json/encode-str data))))
-
-(defn pgarray->set
-  [v]
-  (set (.getArray ^PgArray v)))
-
-(defn pgarray->vector
-  [v]
-  (vec (.getArray ^PgArray v)))
-
+    (.setValue (json/write-str data))))
 
 ;; --- Locks
 

backend/src/app/emails.clj

@@ -7,12 +7,13 @@
 (ns app.emails
   "Main api for send emails."
   (:require
+   [app.common.logging :as l]
+   [app.common.pprint :as pp]
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.db.sql :as sql]
    [app.util.emails :as emails]
-   [app.util.logging :as l]
    [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
@@ -54,10 +55,10 @@
 (defn allow-send-emails?
   [conn profile]
   (when-not (:is-muted profile false)
-    (let [complaint-threshold (cfg/get :profile-complaint-threshold)
-          complaint-max-age   (cfg/get :profile-complaint-max-age)
-          bounce-threshold    (cfg/get :profile-bounce-threshold)
-          bounce-max-age      (cfg/get :profile-bounce-max-age)
+    (let [complaint-threshold (cf/get :profile-complaint-threshold)
+          complaint-max-age   (cf/get :profile-complaint-max-age)
+          bounce-threshold    (cf/get :profile-bounce-threshold)
+          bounce-max-age      (cf/get :profile-bounce-max-age)
 
           {:keys [complaints bounces] :as result}
           (db/exec-one! conn [sql:profile-complaint-report
@@ -66,8 +67,8 @@
                               (:id profile)
                               (db/interval bounce-max-age)])]
 
-      (and (< complaints complaint-threshold)
-           (< bounces bounce-threshold)))))
+      (and (< (or complaints 0) complaint-threshold)
+           (< (or bounces 0) bounce-threshold)))))
 
 (defn has-complaint-reports?
   ([conn email] (has-complaint-reports? conn email nil))
@@ -140,19 +141,17 @@
 
 (declare send-console!)
 
-(s/def ::username ::cfg/smtp-username)
-(s/def ::password ::cfg/smtp-password)
-(s/def ::tls ::cfg/smtp-tls)
-(s/def ::ssl ::cfg/smtp-ssl)
-(s/def ::host ::cfg/smtp-host)
-(s/def ::port ::cfg/smtp-port)
-(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
-(s/def ::default-from ::cfg/smtp-default-from)
-(s/def ::enabled ::cfg/smtp-enabled)
+(s/def ::username ::cf/smtp-username)
+(s/def ::password ::cf/smtp-password)
+(s/def ::tls ::cf/smtp-tls)
+(s/def ::ssl ::cf/smtp-ssl)
+(s/def ::host ::cf/smtp-host)
+(s/def ::port ::cf/smtp-port)
+(s/def ::default-reply-to ::cf/smtp-default-reply-to)
+(s/def ::default-from ::cf/smtp-default-from)
 
 (defmethod ig/pre-init-spec ::sendmail-handler [_]
-  (s/keys :req-un [::enabled]
-          :opt-un [::username
+  (s/keys :opt-un [::username
                    ::password
                    ::tls
                    ::ssl
@@ -164,19 +163,28 @@
 (defmethod ig/init-key ::sendmail-handler
   [_ cfg]
   (fn [{:keys [props] :as task}]
-    (if (:enabled cfg)
-      (emails/send! cfg props)
-      (send-console! cfg props))))
+    (let [enabled? (or (contains? cf/flags :smtp)
+                       (cf/get :smtp-enabled)
+                       (:enabled task))]
+      (when enabled?
+        (emails/send! cfg props))
+
+      (when (contains? cf/flags :log-emails)
+        (send-console! cfg props)))))
 
 (defn- send-console!
-  [cfg email]
-  (let [baos (java.io.ByteArrayOutputStream.)
-        mesg (emails/smtp-message cfg email)]
-    (.writeTo mesg baos)
-    (let [out (with-out-str
-                (println "email console dump:")
-                (println "******** start email" (:id email) "**********")
-                (println (.toString baos))
-                (println "******** end email "(:id email) "**********"))]
-      (l/info :email out))))
+  [_ email]
+  (let [body (:body email)
+        out  (with-out-str
+               (println "email console dump:")
+               (println "******** start email" (:id email) "**********")
+               (pp/pprint (dissoc email :body))
+               (if (string? body)
+                 (println body)
+                 (println (->> body
+                               (filter #(= "text/plain" (:type %)))
+                               (map :content)
+                               first)))
+               (println "******** end email" (:id email) "**********"))]
+    (l/info ::l/raw out)))
 

backend/src/app/http.clj

@@ -7,153 +7,173 @@
 (ns app.http
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
+   [app.common.logging :as l]
+   [app.common.transit :as t]
    [app.http.errors :as errors]
    [app.http.middleware :as middleware]
    [app.metrics :as mtx]
-   [app.util.logging :as l]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
-   [reitit.ring :as rr]
-   [ring.adapter.jetty9 :as jetty])
-  (:import
-   org.eclipse.jetty.server.Server
-   org.eclipse.jetty.server.handler.ErrorHandler
-   org.eclipse.jetty.server.handler.StatisticsHandler))
+   [reitit.core :as r]
+   [reitit.middleware :as rr]
+   [yetti.adapter :as yt]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
-(declare router-handler)
+(declare wrap-router)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP SERVER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::handler fn?)
 (s/def ::router some?)
-(s/def ::ws (s/map-of ::us/string fn?))
-(s/def ::port ::us/integer)
-(s/def ::name ::us/string)
+(s/def ::port integer?)
+(s/def ::host string?)
+(s/def ::name string?)
 
-(defmethod ig/pre-init-spec ::server [_]
-  (s/keys :req-un [::port]
-          :opt-un [::ws ::name ::mtx/metrics ::router ::handler]))
+(s/def ::max-body-size integer?)
+(s/def ::max-multipart-body-size integer?)
+(s/def ::io-threads integer?)
+(s/def ::worker-threads integer?)
 
 (defmethod ig/prep-key ::server
   [_ cfg]
-  (merge {:name "http"} (d/without-nils cfg)))
+  (merge {:name "http"
+          :port 6060
+          :host "0.0.0.0"
+          :max-body-size (* 1024 1024 30)             ; 30 MiB
+          :max-multipart-body-size (* 1024 1024 120)} ; 120 MiB
+         (d/without-nils cfg)))
+
+(defmethod ig/pre-init-spec ::server [_]
+  (s/and
+   (s/keys :req-un [::port ::host ::name ::max-body-size ::max-multipart-body-size]
+           :opt-un [::router ::handler ::io-threads ::worker-threads ::wrk/executor])
+   (fn [cfg]
+     (or (contains? cfg :router)
+         (contains? cfg :handler)))))
 
 (defmethod ig/init-key ::server
-  [_ {:keys [handler router ws port name metrics] :as opts}]
-  (l/info :msg "starting http server" :port port :name name)
-  (let [pre-start (fn [^Server server]
-                    (let [handler (doto (ErrorHandler.)
-                                    (.setShowStacks true)
-                                    (.setServer server))]
-                      (.setErrorHandler server ^ErrorHandler handler)
-                      (when metrics
-                        (let [stats (StatisticsHandler.)]
-                          (.setHandler ^StatisticsHandler stats (.getHandler server))
-                          (.setHandler server stats)
-                          (mtx/instrument-jetty! (:registry metrics) stats)))))
+  [_ {:keys [handler router port name host] :as cfg}]
+  (l/info :hint "starting http server" :port port :host host :name name)
+  (let [options {:http/port port
+                 :http/host host
+                 :http/max-body-size (:max-body-size cfg)
+                 :http/max-multipart-body-size (:max-multipart-body-size cfg)
+                 :xnio/io-threads (:io-threads cfg)
+                 :xnio/worker-threads (:worker-threads cfg)
+                 :xnio/dispatch (:executor cfg)
+                 :ring/async true}
 
-        options   (merge
-                   {:port port
-                    :h2c? true
-                    :join? false
-                    :allow-null-path-info true
-                    :configurator pre-start}
-                   (when (seq ws)
-                     {:websockets ws}))
+        handler (if (some? router)
+                  (wrap-router router)
 
-        handler   (cond
-                    (fn? handler)  handler
-                    (some? router) (router-handler router)
-                    :else (ex/raise :type :internal
-                                    :code :invalid-argument
-                                    :hint "Missing `handler` or `router` option."))
-
-        server    (jetty/run-jetty handler options)]
-    (assoc opts :server server)))
+                  handler)
+        server  (yt/server handler (d/without-nils options))]
+    (assoc cfg :server (yt/start! server))))
 
 (defmethod ig/halt-key! ::server
-  [_ {:keys [server name port] :as opts}]
-  (l/info :msg "stoping http server"
-          :name name
-          :port port)
-  (jetty/stop-server server))
+  [_ {:keys [server name port] :as cfg}]
+  (l/info :msg "stoping http server" :name name :port port)
+  (yt/stop! server))
 
-(defn- router-handler
+(defn- not-found-handler
+  [_ respond _]
+  (respond (yrs/response 404)))
+
+(defn- wrap-router
   [router]
-  (let [handler (rr/ring-handler router
-                                 (rr/routes
-                                  (rr/create-resource-handler {:path "/"})
-                                  (rr/create-default-handler))
-                                 {:middleware [middleware/server-timing]})]
-    (fn [request]
+  (letfn [(handler [request respond raise]
+            (if-let [match (r/match-by-path router (yrq/path request))]
+              (let [params  (:path-params match)
+                    result  (:result match)
+                    handler (or (:handler result) not-found-handler)
+                    request (-> request
+                                (assoc :path-params params)
+                                (update :params merge params))]
+                (handler request respond raise))
+              (not-found-handler request respond raise)))
+
+          (on-error [cause request respond]
+            (let [{:keys [body] :as response} (errors/handle cause request)]
+              (respond
+               (cond-> response
+                 (map? body)
+                 (-> (update :headers assoc "content-type" "application/transit+json")
+                     (assoc :body (t/encode-str body {:type :json-verbose})))))))]
+
+    (fn [request respond _]
       (try
-        (handler request)
-        (catch Throwable e
-          (try
-            (let [cdata (errors/get-error-context request e)]
-              (l/update-thread-context! cdata)
-              (l/error :hint "unhandled exception"
-                       :message (ex-message e)
-                       :error-id (str (:id cdata))
-                       :cause e))
-            {:status 500 :body "internal server error"}
-            (catch Throwable e
-              (l/error :hint "unhandled exception"
-                       :message (ex-message e)
-                       :cause e)
-              {:status 500 :body "internal server error"})))))))
-
+        (handler request respond #(on-error % request respond))
+        (catch Throwable cause
+          (on-error cause request respond))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Main Handler (Router)
+;; HTTP ROUTER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(s/def ::rpc map?)
-(s/def ::session map?)
 (s/def ::oauth map?)
 (s/def ::storage map?)
 (s/def ::assets map?)
 (s/def ::feedback fn?)
+(s/def ::ws fn?)
+(s/def ::audit-handler fn?)
+(s/def ::awsns-handler fn?)
+(s/def ::session map?)
+(s/def ::rpc-routes (s/nilable vector?))
+(s/def ::debug-routes (s/nilable vector?))
+(s/def ::oidc-routes (s/nilable vector?))
+(s/def ::doc-routes (s/nilable vector?))
 
 (defmethod ig/pre-init-spec ::router [_]
-  (s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
+  (s/keys :req-un [::mtx/metrics
+                   ::ws
+                   ::storage
+                   ::assets
+                   ::session
+                   ::feedback
+                   ::awsns-handler
+                   ::debug-routes
+                   ::oidc-routes
+                   ::audit-handler
+                   ::rpc-routes
+                   ::doc-routes]))
 
 (defmethod ig/init-key ::router
-  [_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
+  [_ {:keys [ws session metrics assets feedback] :as cfg}]
   (rr/router
-   [["/metrics" {:get (:handler metrics)}]
-    ["/assets" {:middleware [[middleware/format-response-body]
-                             [middleware/errors errors/handle]
-                             [middleware/cookies]
-                             (:middleware session)]}
-     ["/by-id/:id" {:get (:objects-handler assets)}]
-     ["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
-     ["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
+   [["" {:middleware [[middleware/server-timing]
+                      [middleware/format-response]
+                      [middleware/params]
+                      [middleware/parse-request]
+                      [middleware/errors errors/handle]
+                      [middleware/restrict-methods]]}
 
-    ["/dbg"
-     ["/error-by-id/:id" {:get (:error-report-handler cfg)}]]
+     ["/metrics" {:handler (:handler metrics)}]
+     ["/assets" {:middleware [(:middleware session)]}
+      ["/by-id/:id" {:handler (:objects-handler assets)}]
+      ["/by-file-media-id/:id" {:handler (:file-objects-handler assets)}]
+      ["/by-file-media-id/:id/thumbnail" {:handler (:file-thumbnails-handler assets)}]]
 
-    ["/webhooks"
-     ["/sns" {:post (:sns-webhook cfg)}]]
+     (:debug-routes cfg)
 
-    ["/api" {:middleware [
-                          ;; Temporary disabled
-                          #_[middleware/etag]
-                          [middleware/format-response-body]
-                          [middleware/params]
-                          [middleware/multipart-params]
-                          [middleware/keyword-params]
-                          [middleware/parse-request-body]
-                          [middleware/errors errors/handle]
-                          [middleware/cookies]]}
+     ["/webhooks"
+      ["/sns" {:handler (:awsns-handler cfg)
+               :allowed-methods #{:post}}]]
 
-     ["/feedback" {:middleware [(:middleware session)]
-                   :post feedback}]
+     ["/ws/notifications" {:middleware [(:middleware session)]
+                           :handler ws
+                           :allowed-methods #{:get}}]
 
-     ["/auth/oauth/:provider" {:post (:handler oauth)}]
-     ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
+     ["/api" {:middleware [[middleware/cors]
+                           [(:middleware session)]]}
+      ["/audit/events" {:handler (:audit-handler cfg)
+                        :allowed-methods #{:post}}]
+      ["/feedback" {:handler feedback
+                    :allowed-methods #{:post}}]
+      (:doc-routes cfg)
+      (:oidc-routes cfg)
+      (:rpc-routes cfg)]]]))
 
-     ["/rpc" {:middleware [(:middleware session)]}
-      ["/query/:type" {:get (:query-handler rpc)
-                       :post (:query-handler rpc)}]
-      ["/mutation/:type" {:post (:mutation-handler rpc)}]]]]))

backend/src/app/http/assets.clj

@@ -14,8 +14,12 @@
    [app.metrics :as mtx]
    [app.storage :as sto]
    [app.util.time :as dt]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.response :as yrs]))
 
 (def ^:private cache-max-age
   (dt/duration {:hours 24}))
@@ -25,73 +29,90 @@
 
 (defn coerce-id
   [id]
-  (let [res (us/uuid-conformer id)]
+  (let [res (parse-uuid id)]
     (when-not (uuid? res)
       (ex/raise :type :not-found
                 :hint "object not found"))
     res))
 
 (defn- get-file-media-object
-  [{:keys [pool] :as storage} id]
-  (let [id   (coerce-id id)
-        mobj (db/exec-one! pool ["select * from file_media_object where id=?" id])]
-    (when-not mobj
-      (ex/raise :type :not-found
-                :hint "object does not found"))
-      mobj))
+  [{:keys [pool executor] :as storage} id]
+  (px/with-dispatch executor
+    (let [id   (coerce-id id)
+          mobj (db/exec-one! pool ["select * from file_media_object where id=?" id])]
+      (when-not mobj
+        (ex/raise :type :not-found
+                  :hint "object does not found"))
+      mobj)))
 
 (defn- serve-object
+  "Helper function that returns the appropriate response depending on
+  the storage object backend type."
   [{:keys [storage] :as cfg} obj]
   (let [mdata   (meta obj)
         backend (sto/resolve-backend storage (:backend obj))]
     (case (:type backend)
       :db
-      {:status 200
-       :headers {"content-type" (:content-type mdata)
-                 "cache-control" (str "max-age=" (inst-ms cache-max-age))}
-       :body (sto/get-object-bytes storage obj)}
+      (p/let [body (sto/get-object-bytes storage obj)]
+        (yrs/response :status  200
+                      :body    body
+                      :headers {"content-type" (:content-type mdata)
+                                "cache-control" (str "max-age=" (inst-ms cache-max-age))}))
 
       :s3
-      (let [url (sto/get-object-url storage obj {:max-age signature-max-age})]
-        {:status 307
-         :headers {"location" (str url)
-                   "x-host"   (:host url)
-                   "cache-control" (str "max-age=" (inst-ms cache-max-age))}
-         :body ""})
+      (p/let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
+        (yrs/response :status  307
+                      :headers {"location" (str url)
+                                "x-host"   (cond-> host port (str ":" port))
+                                "cache-control" (str "max-age=" (inst-ms cache-max-age))}))
 
       :fs
-      (let [purl (u/uri (:assets-path cfg))
-            purl (u/join purl (sto/object->relative-path obj))]
-        {:status 204
-         :headers {"x-accel-redirect" (:path purl)
-                   "content-type" (:content-type mdata)
-                   "cache-control" (str "max-age=" (inst-ms cache-max-age))}
-         :body ""}))))
-
-(defn- generic-handler
-  [{:keys [storage] :as cfg} _request id]
-  (let [obj (sto/get-object storage id)]
-    (if obj
-      (serve-object cfg obj)
-      {:status 404 :body ""})))
+      (p/let [purl (u/uri (:assets-path cfg))
+              purl (u/join purl (sto/object->relative-path obj))]
+        (yrs/response :status  204
+                      :headers {"x-accel-redirect" (:path purl)
+                                "content-type" (:content-type mdata)
+                                "cache-control" (str "max-age=" (inst-ms cache-max-age))})))))
 
 (defn objects-handler
-  [cfg request]
-  (let [id (get-in request [:path-params :id])]
-    (generic-handler cfg request (coerce-id id))))
+  "Handler that servers storage objects by id."
+  [{:keys [storage executor] :as cfg} request respond raise]
+  (-> (px/with-dispatch executor
+        (p/let [id  (get-in request [:path-params :id])
+                id  (coerce-id id)
+                obj (sto/get-object storage id)]
+          (if obj
+            (serve-object cfg obj)
+            (yrs/response 404))))
+
+      (p/bind p/wrap)
+      (p/then' respond)
+      (p/catch raise)))
+
+(defn- generic-handler
+  "A generic handler helper/common code for file-media based handlers."
+  [{:keys [storage] :as cfg} request kf]
+  (p/let [id   (get-in request [:path-params :id])
+          mobj (get-file-media-object storage id)
+          obj  (sto/get-object storage (kf mobj))]
+    (if obj
+      (serve-object cfg obj)
+      (yrs/response 404))))
 
 (defn file-objects-handler
-  [{:keys [storage] :as cfg} request]
-  (let [id   (get-in request [:path-params :id])
-        mobj (get-file-media-object storage id)]
-    (generic-handler cfg request (:media-id mobj))))
+  "Handler that serves storage objects by file media id."
+  [cfg request respond raise]
+  (-> (generic-handler cfg request :media-id)
+      (p/then respond)
+      (p/catch raise)))
 
 (defn file-thumbnails-handler
-  [{:keys [storage] :as cfg} request]
-  (let [id   (get-in request [:path-params :id])
-        mobj (get-file-media-object storage id)]
-    (generic-handler cfg request (or (:thumbnail-id mobj) (:media-id mobj)))))
-
+  "Handler that serves storage objects by thumbnail-id and quick
+  fallback to file-media-id if no thumbnail is available."
+  [cfg request respond raise]
+  (-> (generic-handler cfg request #(or (:thumbnail-id %) (:media-id %)))
+      (p/then respond)
+      (p/catch raise)))
 
 ;; --- Initialization
 
@@ -101,10 +122,16 @@
 (s/def ::signature-max-age ::dt/duration)
 
 (defmethod ig/pre-init-spec ::handlers [_]
-  (s/keys :req-un [::storage ::mtx/metrics ::assets-path ::cache-max-age ::signature-max-age]))
+  (s/keys :req-un [::storage
+                   ::wrk/executor
+                   ::mtx/metrics
+                   ::assets-path
+                   ::cache-max-age
+                   ::signature-max-age]))
 
 (defmethod ig/init-key ::handlers
   [_ cfg]
-  {:objects-handler #(objects-handler cfg %)
-   :file-objects-handler #(file-objects-handler cfg %)
-   :file-thumbnails-handler #(file-thumbnails-handler cfg %)})
+  {:objects-handler (partial objects-handler cfg)
+   :file-objects-handler (partial file-objects-handler cfg)
+   :file-thumbnails-handler (partial file-thumbnails-handler cfg)})
+

backend/src/app/http/awsns.clj

@@ -8,33 +8,45 @@
   "AWS SNS webhook handler for bounces."
   (:require
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.db :as db]
    [app.db.sql :as sql]
-   [app.util.http :as http]
-   [app.util.logging :as l]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [jsonista.core :as j]))
+   [jsonista.core :as j]
+   [promesa.exec :as px]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
 (declare parse-json)
+(declare handle-request)
 (declare parse-notification)
 (declare process-report)
 
+(s/def ::http-client fn?)
+
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
+  (s/keys :req-un [::db/pool ::http-client]))
 
 (defmethod ig/init-key ::handler
-  [_ cfg]
-  (fn [request]
-    (let [body  (parse-json (slurp (:body request)))
+  [_ {:keys [executor] :as cfg}]
+  (fn [request respond _]
+    (let [data (-> request yrq/body slurp)]
+      (px/run! executor #(handle-request cfg data)))
+    (respond (yrs/response 200))))
+
+(defn handle-request
+  [{:keys [http-client] :as cfg} data]
+  (try
+    (let [body  (parse-json data)
           mtype (get body "Type")]
       (cond
         (= mtype "SubscriptionConfirmation")
         (let [surl   (get body "SubscribeURL")
               stopic (get body "TopicArn")]
           (l/info :action "subscription received" :topic stopic :url surl)
-          (http/send! {:uri surl :method :post :timeout 10000}))
+          (http-client {:uri surl :method :post :timeout 10000} {:sync? true}))
 
         (= mtype "Notification")
         (when-let [message (parse-json (get body "Message"))]
@@ -43,8 +55,11 @@
 
         :else
         (l/warn :hint "unexpected data received"
-                :report (pr-str body)))
-      {:status 200 :body ""})))
+                :report (pr-str body))))
+
+    (catch Throwable cause
+      (l/error :hint "unexpected exception on awsns"
+               :cause cause))))
 
 (defn- parse-bounce
   [data]
@@ -173,14 +188,14 @@
 
 (defn- process-report
   [cfg {:keys [type profile-id] :as report}]
-  (l/trace :action "procesing report" :report (pr-str report))
+  (l/trace :action "processing report" :report (pr-str report))
   (cond
     ;; In this case we receive a bounce/complaint notification without
     ;; confirmed identity, we just emit a warning but do nothing about
     ;; it because this is not a normal case. All notifications should
     ;; come with profile identity.
     (nil? profile-id)
-    (l/warn :msg "a notification without identity recevied from AWS"
+    (l/warn :msg "a notification without identity received from AWS"
             :report (pr-str report))
 
     (= "bounce" type)

backend/src/app/http/client.clj

@@ -0,0 +1,30 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.client
+  "Http client abstraction layer."
+  (:require
+   [app.worker :as wrk]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [java-http-clj.core :as http]))
+
+(defmethod ig/pre-init-spec :app.http/client [_]
+  (s/keys :req-un [::wrk/executor]))
+
+(defmethod ig/init-key :app.http/client
+  [_ {:keys [executor] :as cfg}]
+  (let [client (http/build-client {:executor executor
+                                   :connect-timeout 30000 ;; 10s
+                                   :follow-redirects :always})]
+    (with-meta
+      (fn send
+        ([req] (send req {}))
+        ([req {:keys [response-type sync?] :or {response-type :string sync? false}}]
+         (if sync?
+           (http/send req {:client client :as response-type})
+           (http/send-async req {:client client :as response-type}))))
+      {::client client})))

backend/src/app/http/debug.clj

@@ -0,0 +1,395 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.debug
+  (:refer-clojure :exclude [error-handler])
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.pprint :as pp]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.http.middleware :as mw]
+   [app.rpc.commands.binfile :as binf]
+   [app.rpc.mutations.files :refer [create-file]]
+   [app.rpc.queries.profile :as profile]
+   [app.util.blob :as blob]
+   [app.util.bytes :as bs]
+   [app.util.template :as tmpl]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [emoji.core :as emj]
+   [integrant.core :as ig]
+   [markdown.core :as md]
+   [markdown.transformers :as mdt]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
+
+;; (selmer.parser/cache-off!)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HELPERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn authorized?
+  [pool {:keys [profile-id]}]
+  (or (= "devenv" (cf/get :host))
+      (let [profile (ex/ignoring (profile/retrieve-profile-data pool profile-id))
+            admins  (or (cf/get :admins) #{})]
+        (contains? admins (:email profile)))))
+
+(defn prepare-response
+  [body]
+  (let [headers {"content-type" "application/transit+json"}]
+    (yrs/response :status 200 :body body :headers headers)))
+
+(defn prepare-download-response
+  [body filename]
+  (let [headers {"content-disposition" (str "attachment; filename=" filename)
+                 "content-type" "application/octet-stream"}]
+    (yrs/response :status 200 :body body :headers headers)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; INDEX
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn index-handler
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+  (yrs/response :status  200
+                :headers {"content-type" "text/html"}
+                :body    (-> (io/resource "templates/debug.tmpl")
+                             (tmpl/render {}))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FILE CHANGES
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def sql:retrieve-range-of-changes
+  "select revn, changes from file_change where file_id=? and revn >= ? and revn <= ? order by revn")
+
+(def sql:retrieve-single-change
+  "select revn, changes, data from file_change where file_id=? and revn = ?")
+
+(defn- retrieve-file-data
+  [{:keys [pool]} {:keys [params profile-id] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (let [file-id  (some-> params :file-id parse-uuid)
+        revn     (some-> params :revn parse-long)
+        filename (str file-id)]
+
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments))
+
+    (let [data (if (integer? revn)
+                 (some-> (db/exec-one! pool [sql:retrieve-single-change file-id revn]) :data)
+                 (some-> (db/get-by-id pool :file file-id) :data))]
+
+      (when-not data
+        (ex/raise :type :not-found
+                  :code :enpty-data
+                  :hint "empty response"))
+      (cond
+        (contains? params :download)
+        (prepare-download-response data filename)
+
+        (contains? params :clone)
+        (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
+              data       (some-> data blob/decode)]
+          (create-file pool {:id (uuid/next)
+                             :name (str "Cloned file: " filename)
+                             :project-id project-id
+                             :profile-id profile-id
+                             :data data})
+          (yrs/response 201 "OK CREATED"))
+
+        :else
+        (prepare-response (some-> data blob/decode))))))
+
+(defn- is-file-exists?
+  [pool id]
+  (let [sql "select exists (select 1 from file where id=?) as exists;"]
+    (-> (db/exec-one! pool [sql id]) :exists)))
+
+(defn- upload-file-data
+  [{:keys [pool]} {:keys [profile-id params] :as request}]
+  (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
+        data       (some-> params :file :path bs/read-as-bytes blob/decode)]
+
+    (if (and data project-id)
+      (let [fname      (str "Imported file *: " (dt/now))
+            overwrite? (contains? params :overwrite?)
+            file-id    (or (and overwrite? (ex/ignoring (-> params :file :filename parse-uuid)))
+                           (uuid/next))]
+
+        (if (and overwrite? file-id
+                 (is-file-exists? pool file-id))
+          (do
+            (db/update! pool :file
+                        {:data (blob/encode data)}
+                        {:id file-id})
+            (yrs/response 200 "OK UPDATED"))
+
+          (do
+            (create-file pool {:id file-id
+                               :name fname
+                               :project-id project-id
+                               :profile-id profile-id
+                               :data data})
+            (yrs/response 201 "OK CREATED"))))
+
+      (yrs/response 500 "ERROR"))))
+
+(defn file-data-handler
+  [cfg request]
+  (case (yrq/method request)
+    :get (retrieve-file-data cfg request)
+    :post (upload-file-data cfg request)
+    (ex/raise :type :http
+              :code :method-not-found)))
+
+(defn file-changes-handler
+  [{:keys [pool]} {:keys [params] :as request}]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+
+  (letfn [(retrieve-changes [file-id revn]
+            (if (str/includes? revn ":")
+              (let [[start end] (->> (str/split revn #":")
+                                     (map str/trim)
+                                     (map parse-long))]
+                (some->> (db/exec! pool [sql:retrieve-range-of-changes file-id start end])
+                         (map :changes)
+                         (map blob/decode)
+                         (mapcat identity)
+                         (vec)))
+
+              (if-let [revn (parse-long revn)]
+                (let [item (db/exec-one! pool [sql:retrieve-single-change file-id revn])]
+                  (some-> item :changes blob/decode vec))
+                (ex/raise :type :validation :code :invalid-arguments))))]
+
+    (let [file-id  (some-> params :id parse-uuid)
+          revn     (or (some-> params :revn parse-long) "latest")
+          filename (str file-id)]
+
+      (when (or (not file-id) (not revn))
+        (ex/raise :type :validation
+                  :code :invalid-arguments
+                  :hint "missing arguments"))
+
+      (let [data (retrieve-changes file-id revn)]
+        (if (contains? params :download)
+          (prepare-download-response data filename)
+          (prepare-response data))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ERROR BROWSER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn error-handler
+  [{:keys [pool]} request]
+  (letfn [(parse-id [request]
+            (let [id (get-in request [:path-params :id])
+                  id (parse-uuid id)]
+              (when (uuid? id)
+                id)))
+
+          (retrieve-report [id]
+            (ex/ignoring
+             (some-> (db/get-by-id pool :server-error-report id) :content db/decode-transit-pgobject)))
+
+          (render-template [report]
+            (let [context (dissoc report
+                                  :trace :cause :params :data :spec-problems
+                                  :spec-explain :spec-value :error :explain :hint)
+                  params  {:context       (pp/pprint-str context :width 200)
+                           :hint          (:hint report)
+                           :spec-explain  (:spec-explain report)
+                           :spec-problems (:spec-problems report)
+                           :spec-value    (:spec-value report)
+                           :data          (:data report)
+                           :trace         (or (:trace report)
+                                              (some-> report :error :trace))
+                           :params        (:params report)}]
+              (-> (io/resource "templates/error-report.tmpl")
+                  (tmpl/render params))))]
+
+    (when-not (authorized? pool request)
+      (ex/raise :type :authentication
+                :code :only-admins-allowed))
+
+    (let [result (some-> (parse-id request)
+                         (retrieve-report)
+                         (render-template))]
+      (if result
+        (yrs/response :status 200
+                      :body result
+                      :headers {"content-type" "text/html; charset=utf-8"
+                                "x-robots-tag" "noindex"})
+        (yrs/response 404 "not found")))))
+
+(def sql:error-reports
+  "select id, created_at from server_error_report order by created_at desc limit 100")
+
+(defn error-list-handler
+  [{:keys [pool]} request]
+  (when-not (authorized? pool request)
+    (ex/raise :type :authentication
+              :code :only-admins-allowed))
+  (let [items (db/exec! pool [sql:error-reports])
+        items (map #(update % :created-at dt/format-instant :rfc1123) items)]
+    (yrs/response :status 200
+                  :body (-> (io/resource "templates/error-list.tmpl")
+                            (tmpl/render {:items items}))
+                  :headers {"content-type" "text/html; charset=utf-8"
+                            "x-robots-tag" "noindex"})))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; EXPORT/IMPORT
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn export-handler
+  [{:keys [pool] :as cfg} {:keys [params profile-id] :as request}]
+
+  (let [file-ids (->> (:file-ids params)
+                      (remove empty?)
+                      (mapv parse-uuid))
+        libs?    (contains? params :includelibs)
+        clone?   (contains? params :clone)
+        embed?   (contains? params :embedassets)]
+
+    (when-not (seq file-ids)
+      (ex/raise :type :validation
+                :code :missing-arguments))
+
+    (let [path (-> cfg
+                   (assoc ::binf/file-ids file-ids)
+                   (assoc ::binf/embed-assets? embed?)
+                   (assoc ::binf/include-libraries? libs?)
+                   (binf/export!))]
+      (if clone?
+        (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)]
+          (binf/import!
+           (assoc cfg
+                  ::binf/input path
+                  ::binf/overwrite? false
+                  ::binf/ignore-index-errors? true
+                  ::binf/profile-id profile-id
+                  ::binf/project-id project-id))
+
+          (yrs/response
+           :status  200
+           :headers {"content-type" "text/plain"}
+           :body    "OK CLONED"))
+
+        (yrs/response
+         :status  200
+         :headers {"content-type" "application/octet-stream"
+                   "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}
+         :body    (io/input-stream path))))))
+
+
+(defn import-handler
+  [{:keys [pool] :as cfg} {:keys [params profile-id] :as request}]
+  (when-not (contains? params :file)
+    (ex/raise :type :validation
+              :code :missing-upload-file
+              :hint "missing upload file"))
+
+  (let [project-id (some-> (profile/retrieve-additional-data pool profile-id) :default-project-id)
+        overwrite? (contains? params :overwrite)
+        migrate? (contains? params :migrate)
+        ignore-index-errors? (contains? params :ignore-index-errors)]
+
+    (when-not project-id
+      (ex/raise :type :validation
+                :code :missing-project
+                :hint "project not found"))
+
+    (binf/import!
+     (assoc cfg
+            ::binf/input (-> params :file :path)
+            ::binf/overwrite? overwrite?
+            ::binf/migrate? migrate?
+            ::binf/ignore-index-errors? ignore-index-errors?
+            ::binf/profile-id profile-id
+            ::binf/project-id project-id))
+
+    (yrs/response
+     :status  200
+     :headers {"content-type" "text/plain"}
+     :body    "OK")))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; OTHER SMALL VIEWS/HANDLERS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn health-handler
+  "Mainly a task that performs a health check."
+  [{:keys [pool]} _]
+  (db/with-atomic [conn pool]
+    (db/exec-one! conn ["select count(*) as count from server_prop;"])
+    (yrs/response 200 "OK")))
+
+(defn changelog-handler
+  [_ _]
+  (letfn [(transform-emoji [text state]
+            [(emj/emojify text) state])
+          (md->html [text]
+            (md/md-to-html-string text :replacement-transformers (into [transform-emoji] mdt/transformer-vector)))]
+    (if-let [clog (io/resource "changelog.md")]
+      (yrs/response :status 200
+                    :headers {"content-type" "text/html; charset=utf-8"}
+                    :body (-> clog slurp md->html))
+      (yrs/response :status 404 :body "NOT FOUND"))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; INIT
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def with-authorization
+  {:compile
+   (fn [& _]
+     (fn [handler pool]
+       (fn [request respond raise]
+         (if (authorized? pool request)
+           (handler request respond raise)
+           (raise (ex/error :type :authentication
+                            :code :only-admins-allowed))))))})
+
+
+(s/def ::session map?)
+
+(defmethod ig/pre-init-spec ::routes [_]
+  (s/keys :req-un [::db/pool ::wrk/executor ::session]))
+
+(defmethod ig/init-key ::routes
+  [_ {:keys [session pool executor] :as cfg}]
+  ["/dbg" {:middleware [[(:middleware session)]
+                        [with-authorization pool]
+                        [mw/with-promise-async executor]
+                        [mw/with-config cfg]]}
+   ["" {:handler index-handler}]
+   ["/health" {:handler health-handler}]
+   ["/changelog" {:handler changelog-handler}]
+   ;; ["/error-by-id/:id" {:handler error-handler}]
+   ["/error/:id" {:handler error-handler}]
+   ["/error" {:handler error-list-handler}]
+   ["/file/export" {:handler export-handler}]
+   ["/file/import" {:handler import-handler}]
+   ["/file/data" {:handler file-data-handler}]
+   ["/file/changes" {:handler file-changes-handler}]])

backend/src/app/http/errors.clj

@@ -8,30 +8,33 @@
   "A errors handling for the http server."
   (:require
    [app.common.exceptions :as ex]
-   [app.common.uuid :as uuid]
-   [app.util.logging :as l]
+   [app.common.logging :as l]
+   [app.common.spec :as us]
+   [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [expound.alpha :as expound]))
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
-(defn- explain-error
-  [error]
-  (with-out-str
-    (expound/printer (:data error))))
+(def ^:dynamic *context* {})
 
-(defn get-error-context
-  [request error]
-  (let [edata (ex-data error)]
-    (merge
-     {:id      (uuid/next)
-      :path    (:uri request)
-      :method  (:request-method request)
-      :params  (:params request)
-      :data    edata}
-     (let [headers (:headers request)]
-       {:user-agent (get headers "user-agent")
-        :frontend-version (get headers "x-frontend-version" "unknown")})
-     (when (and (map? edata) (:data edata))
-       {:explain (explain-error edata)}))))
+(defn- parse-client-ip
+  [request]
+  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
+      (yrq/get-header request "x-real-ip")
+      (yrq/remote-addr request)))
+
+(defn get-context
+  [request]
+  (merge
+   *context*
+   {:path          (:path request)
+    :method        (:method request)
+    :params        (:params request)
+    :ip-addr       (parse-client-ip request)
+    :profile-id    (:profile-id request)}
+   (let [headers (:headers request)]
+     {:user-agent (get headers "user-agent")
+      :frontend-version (get headers "x-frontend-version" "unknown")})))
 
 (defmulti handle-exception
   (fn [err & _rest]
@@ -41,108 +44,115 @@
 
 (defmethod handle-exception :authentication
   [err _]
-  {:status 401 :body (ex-data err)})
-
+  (yrs/response 401 (ex-data err)))
 
 (defmethod handle-exception :restriction
   [err _]
-  {:status 400 :body (ex-data err)})
+  (yrs/response 400 (ex-data err)))
 
 (defmethod handle-exception :validation
-  [err req]
-  (let [header (get-in req [:headers "accept"])
-        edata  (ex-data err)]
-    (if (and (= :spec-validation (:code edata))
-             (str/starts-with? header "text/html"))
-      {:status 400
-       :headers {"content-type" "text/html"}
-       :body (str "<pre style='font-size:16px'>"
-                  (explain-error edata)
-                  "</pre>\n")}
-      {:status 400
-       :body   (cond-> edata
-                 (map? (:data edata))
-                 (-> (assoc :explain (explain-error edata))
-                     (dissoc :data)))})))
+  [err _]
+  (let [{:keys [code] :as data} (ex-data err)]
+    (cond
+      (= code :spec-validation)
+      (let [explain (us/pretty-explain data)]
+        (yrs/response :status 400
+                      :body   (-> data
+                                  (dissoc ::s/problems ::s/value)
+                                  (cond-> explain (assoc :explain explain)))))
+
+      (= code :request-body-too-large)
+      (yrs/response :status 413 :body data)
+
+      :else
+      (yrs/response :status 400 :body data))))
 
 (defmethod handle-exception :assertion
   [error request]
   (let [edata (ex-data error)
-        cdata (get-error-context request error)]
-    (l/update-thread-context! cdata)
-    (l/error :hint "internal error: assertion"
-             :error-id (str (:id cdata))
+        explain (us/pretty-explain edata)]
+    (l/error ::l/raw (str (ex-message error) "\n" explain)
+             ::l/context (get-context request)
              :cause error)
-
-    {:status 500
-     :body {:type :server-error
-            :code :assertion
-            :data (-> edata
-                      (assoc :explain (explain-error edata))
-                      (dissoc :data))}}))
+    (yrs/response :status 500
+                  :body   {:type :server-error
+                           :code :assertion
+                           :data (-> edata
+                                     (dissoc ::s/problems ::s/value ::s/spec)
+                                     (cond-> explain (assoc :explain explain)))})))
 
 (defmethod handle-exception :not-found
   [err _]
-  {:status 404 :body (ex-data err)})
+  (yrs/response 404 (ex-data err)))
+
+(defmethod handle-exception org.postgresql.util.PSQLException
+  [error request]
+  (let [state (.getSQLState ^java.sql.SQLException error)]
+    (l/error ::l/raw (ex-message error)
+             ::l/context (get-context request)
+             :cause error)
+    (cond
+      (= state "57014")
+      (yrs/response 504 {:type :server-error
+                         :code :statement-timeout
+                         :hint (ex-message error)})
+
+      (= state "25P03")
+      (yrs/response 504 {:type :server-error
+                         :code :idle-in-transaction-timeout
+                         :hint (ex-message error)})
+
+      :else
+      (yrs/response 500 {:type :server-error
+                         :code :unexpected
+                         :hint (ex-message error)
+                         :state state}))))
 
 (defmethod handle-exception :default
   [error request]
   (let [edata (ex-data error)]
-    ;; NOTE: this is a special case for the idle-in-transaction error;
-    ;; when it happens, the connection is automatically closed and
-    ;; next-jdbc combines the two errors in a single ex-info. We only
-    ;; need the :handling error, because the :rollback error will be
-    ;; always "connection closed".
-    (if (and (ex/exception? (:rollback edata))
-             (ex/exception? (:handling edata)))
-      (handle-exception (:handling edata) request)
-      (let [cdata (get-error-context request error)]
-        (l/update-thread-context! cdata)
-        (l/error :hint "internal error"
-                 :error-message (ex-message error)
-                 :error-id (str (:id cdata))
-                 :cause error)
-        {:status 500
-         :body {:type :server-error
-                :code :unexpected
-                :hint (ex-message error)
-                :data edata}}))))
-
-(defmethod handle-exception org.postgresql.util.PSQLException
-  [error request]
-  (let [cdata (get-error-context request error)
-        state (.getSQLState ^java.sql.SQLException error)]
-
-    (l/update-thread-context! cdata)
-    (l/error :hint "psql exception"
-             :error-message (ex-message error)
-             :error-id (str (:id cdata))
-             :sql-state state
-             :cause error)
-
     (cond
-      (= state "57014")
-      {:status 504
-       :body {:type :server-timeout
-              :code :statement-timeout
-              :hint (ex-message error)}}
+      ;; This means that exception is not a controlled exception.
+      (nil? edata)
+      (do
+        (l/error ::l/raw (ex-message error)
+                 ::l/context (get-context request)
+                 :cause error)
+        (yrs/response 500 {:type :server-error
+                           :code :unexpected
+                           :hint (ex-message error)}))
 
-      (= state "25P03")
-      {:status 504
-       :body {:type :server-timeout
-              :code :idle-in-transaction-timeout
-              :hint (ex-message error)}}
+      ;; This is a special case for the idle-in-transaction error;
+      ;; when it happens, the connection is automatically closed and
+      ;; next-jdbc combines the two errors in a single ex-info. We
+      ;; only need the :handling error, because the :rollback error
+      ;; will be always "connection closed".
+      (and (ex/exception? (:rollback edata))
+           (ex/exception? (:handling edata)))
+      (handle-exception (:handling edata) request)
 
       :else
-      {:status 500
-       :body {:type :server-error
-              :code :psql-exception
-              :hint (ex-message error)
-              :state state}})))
+      (do
+        (l/error ::l/raw (ex-message error)
+                 ::l/context (get-context request)
+                 :cause error)
+        (yrs/response 500 {:type :server-error
+                           :code :unhandled
+                           :hint (ex-message error)
+                           :data edata})))))
 
 (defn handle
-  [error req]
-  (if (or (instance? java.util.concurrent.CompletionException error)
-          (instance? java.util.concurrent.ExecutionException error))
-    (handle-exception (.getCause ^Throwable error) req)
-    (handle-exception error req)))
+  [cause request]
+  (cond
+    (or (instance? java.util.concurrent.CompletionException cause)
+        (instance? java.util.concurrent.ExecutionException cause))
+    (handle-exception (.getCause ^Throwable cause) request)
+
+    (ex/wrapped? cause)
+    (let [context (meta cause)
+          cause   (deref cause)]
+      (binding [*context* context]
+        (handle-exception cause request)))
+
+    :else
+    (handle-exception cause request)))

backend/src/app/http/feedback.clj

@@ -10,57 +10,67 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.spec :as us]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.emails :as eml]
    [app.rpc.queries.profile :as profile]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
-(declare send-feedback)
+(declare ^:private send-feedback)
+(declare ^:private handler)
 
 (defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
+  (s/keys :req-un [::db/pool ::wrk/executor]))
 
 (defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as scfg}]
-  (let [ftoken   (cfg/get :feedback-token ::no-token)
-        enabled  (cfg/get :feedback-enabled)]
-    (fn [{:keys [profile-id] :as request}]
-      (let [token  (get-in request [:headers "x-feedback-token"])
-            params (d/merge (:params request)
-                            (:body-params request))]
+  [_ {:keys [executor] :as cfg}]
+  (let [enabled? (contains? cf/flags :user-feedback)]
+    (if enabled?
+      (fn [request respond raise]
+        (-> (px/submit! executor #(handler cfg request))
+            (p/then' respond)
+            (p/catch raise)))
+      (fn [_ _ raise]
+        (raise (ex/error :type :validation
+                         :code :feedback-disabled
+                         :hint "feedback module is disabled"))))))
 
-        (when-not enabled
-          (ex/raise :type :validation
-                    :code :feedback-disabled
-                    :hint "feedback module is disabled"))
+(defn- handler
+  [{:keys [pool] :as cfg} {:keys [profile-id] :as request}]
+  (let [ftoken (cf/get :feedback-token ::no-token)
+        token  (yrq/get-header request "x-feedback-token")
+        params (d/merge (:params request)
+                        (:body-params request))]
+    (cond
+      (uuid? profile-id)
+      (let [profile (profile/retrieve-profile-data pool profile-id)
+            params  (assoc params :from (:email profile))]
+        (send-feedback pool profile params))
 
-        (cond
-          (uuid? profile-id)
-          (let [profile (profile/retrieve-profile-data pool profile-id)
-                params  (assoc params :from (:email profile))]
-            (when-not (:is-muted profile)
-              (send-feedback pool profile params)))
+      (= token ftoken)
+      (send-feedback cfg nil params))
 
-          (= token ftoken)
-          (send-feedback scfg nil params))
-
-        {:status 204 :body ""}))))
+    (yrs/response 204)))
 
 (s/def ::content ::us/string)
 (s/def ::from    ::us/email)
 (s/def ::subject ::us/string)
-
 (s/def ::feedback
   (s/keys :req-un [::from ::subject ::content]))
 
-(defn send-feedback
+(defn- send-feedback
   [pool profile params]
   (let [params      (us/conform ::feedback params)
-        destination (cfg/get :feedback-destination)]
+        destination (cf/get :feedback-destination)]
     (eml/send! {::eml/conn pool
                 ::eml/factory eml/feedback
+                :from     destination
                 :to       destination
                 :profile  profile
                 :reply-to (:from params)

backend/src/app/http/middleware.clj

@@ -6,192 +6,209 @@
 
 (ns app.http.middleware
   (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.transit :as t]
-   [app.metrics :as mtx]
+   [app.config :as cf]
    [app.util.json :as json]
-   [app.util.logging :as l]
-   [buddy.core.codecs :as bc]
-   [buddy.core.hash :as bh]
-   [clojure.java.io :as io]
-   [ring.core.protocols :as rp]
-   [ring.middleware.cookies :refer [wrap-cookies]]
-   [ring.middleware.keyword-params :refer [wrap-keyword-params]]
-   [ring.middleware.multipart-params :refer [wrap-multipart-params]]
-   [ring.middleware.params :refer [wrap-params]]))
+   [cuerdas.core :as str]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.adapter :as yt]
+   [yetti.middleware :as ymw]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs])
+  (:import
+   com.fasterxml.jackson.core.io.JsonEOFException
+   io.undertow.server.RequestTooBigException
+   java.io.OutputStream))
 
-(defn wrap-server-timing
+(def server-timing
+  {:name ::server-timing
+   :compile (constantly ymw/wrap-server-timing)})
+
+(def params
+  {:name ::params
+   :compile (constantly ymw/wrap-params)})
+
+(defn wrap-parse-request
   [handler]
-  (let [seconds-from #(float (/ (- (System/nanoTime) %) 1000000000))]
-    (fn [request]
-      (let [start    (System/nanoTime)
-            response (handler request)]
-        (update response :headers
-                (fn [headers]
-                  (assoc headers "Server-Timing" (str "total;dur=" (seconds-from start)))))))))
+  (letfn [(process-request [request]
+            (let [header (yrq/get-header request "content-type")]
+              (cond
+                (str/starts-with? header "application/transit+json")
+                (with-open [is (yrq/body request)]
+                  (let [params (t/read! (t/reader is))]
+                    (-> request
+                        (assoc :body-params params)
+                        (update :params merge params))))
 
-(defn wrap-parse-request-body
+                (str/starts-with? header "application/json")
+                (with-open [is (yrq/body request)]
+                  (let [params (json/read is)]
+                    (-> request
+                        (assoc :body-params params)
+                        (update :params merge params))))
+
+                :else
+                request)))
+
+          (handle-error [raise cause]
+            (cond
+              (instance? RequestTooBigException cause)
+              (raise (ex/error :type :validation
+                               :code :request-body-too-large
+                               :hint (ex-message cause)))
+
+              (instance? JsonEOFException cause)
+              (raise (ex/error :type :validation
+                               :code :malformed-json
+                               :hint (ex-message cause)))
+              :else
+              (raise cause)))]
+
+    (fn [request respond raise]
+      (when-let [request (try
+                           (process-request request)
+                           (catch RuntimeException cause
+                             (handle-error raise (or (.getCause cause) cause)))
+                           (catch Throwable cause
+                             (handle-error raise cause)))]
+        (handler request respond raise)))))
+
+(def parse-request
+  {:name ::parse-request
+   :compile (constantly wrap-parse-request)})
+
+(defn buffered-output-stream
+  "Returns a buffered output stream that ignores flush calls. This is
+  needed because transit-java calls flush very aggresivelly on each
+  object write."
+  [^java.io.OutputStream os ^long chunk-size]
+  (proxy [java.io.BufferedOutputStream] [os (int chunk-size)]
+    ;; Explicitly do not forward flush
+    (flush [])
+    (close []
+      (proxy-super flush)
+      (proxy-super close))))
+
+(def ^:const buffer-size (:xnio/buffer-size yt/defaults))
+
+(defn wrap-format-response
   [handler]
-  (letfn [(parse-transit [body]
-            (let [reader (t/reader body)]
-              (t/read! reader)))
+  (letfn [(transit-streamable-body [data opts]
+            (reify yrs/StreamableResponseBody
+              (-write-body-to-stream [_ _ output-stream]
+                (try
+                  (with-open [bos (buffered-output-stream output-stream buffer-size)]
+                    (let [tw (t/writer bos opts)]
+                      (t/write! tw data)))
 
-          (parse-json [body]
-            (let [reader (io/reader body)]
-              (json/read reader)))
+                  (catch java.io.IOException _cause
+                    ;; Do nothing, EOF means client closes connection abruptly
+                    nil)
+                  (catch Throwable cause
+                    (l/warn :hint "unexpected error on encoding response"
+                            :cause cause))
+                  (finally
+                    (.close ^OutputStream output-stream))))))
 
-          (parse [type body]
-            (try
-              (case type
-                :json (parse-json body)
-                :transit (parse-transit body))
-              (catch Exception e
-                (let [data {:type :parse
-                            :hint "unable to parse request body"
-                            :message (ex-message e)}]
-                  {:status 400
-                   :headers {"content-type" "application/transit+json"}
-                   :body (t/encode-str data {:type :json-verbose})}))))]
+          (format-response [response request]
+            (let [body (yrs/body response)]
+              (if (or (boolean? body) (coll? body))
+                (let [qs   (yrq/query request)
+                      opts (if (or (contains? cf/flags :transit-readable-response)
+                                   (str/includes? qs "transit_verbose"))
+                             {:type :json-verbose}
+                             {:type :json})]
+                  (-> response
+                      (update :headers assoc "content-type" "application/transit+json")
+                      (assoc :body (transit-streamable-body body opts))))
+                response)))
 
-    (fn [{:keys [headers body] :as request}]
-      (let [ctype (get headers "content-type")]
-        (handler
-         (case ctype
-           "application/transit+json"
-           (let [params (parse :transit body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+          (process-response [response request]
+            (cond-> response
+              (map? response) (format-response request)))]
 
-           "application/json"
-           (let [params (parse :json body)]
-             (-> request
-                 (assoc :body-params params)
-                 (update :params merge params)))
+    (fn [request respond raise]
+      (handler request
+               (fn [response]
+                 (let [response (process-response response request)]
+                   (respond response)))
+               raise))))
 
-           request))))))
-
-(def parse-request-body
-  {:name ::parse-request-body
-   :compile (constantly wrap-parse-request-body)})
-
-(defn- transit-streamable-body
-  [data opts]
-  (reify rp/StreamableResponseBody
-    (write-body-to-stream [_ response output-stream]
-      (try
-        (let [tw (t/writer output-stream opts)]
-          (t/write! tw data))
-        (finally
-          (.close ^java.io.OutputStream output-stream))))))
-
-(defn- impl-format-response-body
-  [response _request]
-  (let [body (:body response)
-        opts {:type :json-verbose}]
-    (cond
-      (coll? body)
-      (-> response
-          (update :headers assoc "content-type" "application/transit+json")
-          (assoc :body (transit-streamable-body body opts)))
-
-      ;; ;; Temporary disabled
-      ;; (-> response
-      ;;     (update :headers assoc "content-type" "application/transit+json")
-      ;;     (assoc :body
-      ;;            (if (= :post (:request-method request))
-      ;;              (transit-streamable-body body opts)
-      ;;              (t/encode body opts))))
-
-      (nil? body)
-      (assoc response :status 204 :body "")
-
-      :else
-      response)))
-
-(defn- wrap-format-response-body
-  [handler]
-  (fn [request]
-    (let [response (handler request)]
-      (cond-> response
-        (map? response) (impl-format-response-body request)))))
-
-(def format-response-body
-  {:name ::format-response-body
-   :compile (constantly wrap-format-response-body)})
+(def format-response
+  {:name ::format-response
+   :compile (constantly wrap-format-response)})
 
 (defn wrap-errors
   [handler on-error]
-  (fn [request]
-    (try
-      (handler request)
-      (catch Throwable e
-        (on-error e request)))))
+  (fn [request respond _]
+    (handler request respond (fn [cause]
+                               (-> cause (on-error request) respond)))))
 
 (def errors
   {:name ::errors
    :compile (constantly wrap-errors)})
 
-(def metrics
-  {:name ::metrics
-   :wrap (fn [handler]
-           (mtx/wrap-counter handler {:id "http__requests_counter"
-                                      :help "Absolute http requests counter."}))})
-(def cookies
-  {:name ::cookies
-   :compile (constantly wrap-cookies)})
-
-(def params
-  {:name ::params
-   :compile (constantly wrap-params)})
-
-(def multipart-params
-  {:name ::multipart-params
-   :compile (constantly wrap-multipart-params)})
-
-(def keyword-params
-  {:name ::keyword-params
-   :compile (constantly wrap-keyword-params)})
-
-(def server-timing
-  {:name ::server-timing
-   :compile (constantly wrap-server-timing)})
-
-(defn wrap-etag
+(defn wrap-cors
   [handler]
-  (letfn [(generate-etag [{:keys [body] :as response}]
-            (str "W/\"" (-> body bh/blake2b-128 bc/bytes->hex) "\""))
-          (get-match [{:keys [headers] :as request}]
-            (get headers "if-none-match"))]
-    (fn [request]
-      (let [response (handler request)]
-        (if (= :get (:request-method request))
-          (let [etag     (generate-etag response)
-                match    (get-match request)
-                response (update response :headers #(assoc % "ETag" etag))]
-            (cond-> response
-              (and (string? match)
-                   (= :get (:request-method request))
-                   (= etag match))
-              (-> response
-                  (assoc :body "")
-                  (assoc :status 304))))
-          response)))))
+  (if-not (contains? cf/flags :cors)
+    handler
+    (letfn [(add-headers [headers request]
+              (let [origin (yrq/get-header request "origin")]
+                (-> headers
+                    (assoc "access-control-allow-origin" origin)
+                    (assoc "access-control-allow-methods" "GET,POST,DELETE,OPTIONS,PUT,HEAD,PATCH")
+                    (assoc "access-control-allow-credentials" "true")
+                    (assoc "access-control-expose-headers" "x-requested-with, content-type, cookie")
+                    (assoc "access-control-allow-headers" "x-frontend-version, content-type, accept, x-requested-width"))))
 
-(def etag
-  {:name ::etag
-   :compile (constantly wrap-etag)})
+            (update-response [response request]
+              (update response :headers add-headers request))]
 
-(defn activity-logger
-  [handler]
-  (let [logger "penpot.profile-activity"]
-    (fn [{:keys [headers] :as request}]
-      (let [ip-addr    (get headers "x-forwarded-for")
-            profile-id (:profile-id request)
-            qstring    (:query-string request)]
-        (l/info ::l/async true
-                ::l/logger logger
-                :ip-addr ip-addr
-                :profile-id profile-id
-                :uri (str (:uri request) (when qstring (str "?" qstring)))
-                :method (name (:request-method request)))
-        (handler request)))))
+      (fn [request respond raise]
+        (if (= (yrq/method request) :options)
+          (-> (yrs/response 200)
+              (update-response request)
+              (respond))
+          (handler request
+                   (fn [response]
+                     (respond (update-response response request)))
+                   raise))))))
+
+(def cors
+  {:name ::cors
+   :compile (constantly wrap-cors)})
+
+(defn compile-restrict-methods
+  [data _]
+  (when-let [allowed (:allowed-methods data)]
+    (fn [handler]
+      (fn [request respond raise]
+        (let [method (yrq/method request)]
+          (if (contains? allowed method)
+            (handler request respond raise)
+            (respond (yrs/response 405))))))))
+
+(def restrict-methods
+  {:name ::restrict-methods
+   :compile compile-restrict-methods})
+
+(def with-promise-async
+  {:compile
+   (fn [& _]
+     (fn [handler executor]
+       (fn [request respond raise]
+         (-> (px/submit! executor #(handler request))
+             (p/bind p/wrap)
+             (p/then respond)
+             (p/catch raise)))))})
+
+(def with-config
+  {:compile
+   (fn [& _]
+     (fn [handler config]
+       (fn
+         ([request] (handler config request))
+         ([request respond raise] (handler config request respond raise)))))})

backend/src/app/http/oauth.clj

@@ -1,371 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.http.oauth
-  (:require
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.common.uri :as u]
-   [app.config :as cf]
-   [app.db :as db]
-   [app.loggers.audit :as audit]
-   [app.rpc.queries.profile :as profile]
-   [app.util.http :as http]
-   [app.util.logging :as l]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.set :as set]
-   [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
-   [integrant.core :as ig]))
-
-(defn- build-redirect-uri
-  [{:keys [provider] :as cfg}]
-  (let [public (u/uri (:public-uri cfg))]
-    (str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
-
-(defn- build-auth-uri
-  [{:keys [provider] :as cfg} state]
-  (let [params {:client_id (:client-id provider)
-                :redirect_uri (build-redirect-uri cfg)
-                :response_type "code"
-                :state state
-                :scope (str/join " " (:scopes provider []))}
-        query  (u/map->query-string params)]
-    (-> (u/uri (:auth-uri provider))
-        (assoc :query query)
-        (str))))
-
-(defn retrieve-access-token
-  [{:keys [provider] :as cfg} code]
-  (try
-    (let [params {:client_id (:client-id provider)
-                  :client_secret (:client-secret provider)
-                  :code code
-                  :grant_type "authorization_code"
-                  :redirect_uri (build-redirect-uri cfg)}
-          req    {:method :post
-                  :headers {"content-type" "application/x-www-form-urlencoded"}
-                  :uri (:token-uri provider)
-                  :body (u/map->query-string params)}
-          res    (http/send! req)]
-      (when (= 200 (:status res))
-        (let [data (json/read-str (:body res))]
-          {:token (get data "access_token")
-           :type  (get data "token_type")})))
-    (catch Exception e
-      (l/error :hint "unexpected error on retrieve-access-token"
-               :cause e)
-      nil)))
-
-(defn- retrieve-user-info
-  [{:keys [provider] :as cfg} tdata]
-  (try
-    (let [req {:uri (:user-uri provider)
-               :headers {"Authorization" (str (:type tdata) " " (:token tdata))}
-               :timeout 6000
-               :method :get}
-          res (http/send! req)]
-
-      (when (= 200 (:status res))
-        (let [info (json/read-str (:body res) :key-fn keyword)]
-          {:backend (:name provider)
-           :email (:email info)
-           :fullname (:name info)
-           :props (dissoc info :name :email)})))
-
-    (catch Exception e
-      (l/error :hint "unexpected exception on retrieve-user-info"
-               :cause e)
-      nil)))
-
-(s/def ::backend ::us/not-empty-string)
-(s/def ::email ::us/not-empty-string)
-(s/def ::fullname ::us/not-empty-string)
-(s/def ::props (s/map-of ::us/keyword any?))
-
-(s/def ::info
-  (s/keys :req-un [::backend
-                   ::email
-                   ::fullname
-                   ::props]))
-
-(defn retrieve-info
-  [{:keys [tokens provider] :as cfg} request]
-  (let [state  (get-in request [:params :state])
-        state  (tokens :verify {:token state :iss :oauth})
-        info   (some->> (get-in request [:params :code])
-                        (retrieve-access-token cfg)
-                        (retrieve-user-info cfg))]
-
-    (when-not (s/valid? ::info info)
-      (l/warn :hint "received incomplete profile info object (please set correct scopes)"
-              :info (pr-str info))
-      (ex/raise :type :internal
-                :code :unable-to-auth
-                :hint "no user info"))
-
-    ;; If the provider is OIDC, we can proceed to check
-    ;; roles if they are defined.
-    (when (and (= "oidc" (:name provider))
-               (seq (:roles provider)))
-      (let [provider-roles (into #{} (:roles provider))
-            profile-roles  (let [attr  (cf/get :oidc-roles-attr :roles)
-                                 roles (get info attr)]
-                             (cond
-                               (string? roles) (into #{} (str/words roles))
-                               (vector? roles) (into #{} roles)
-                               :else #{}))]
-
-        ;; check if profile has a configured set of roles
-        (when-not (set/subset? provider-roles profile-roles)
-          (ex/raise :type :internal
-                    :code :unable-to-auth
-                    :hint "not enought permissions"))))
-
-    (cond-> info
-      (some? (:invitation-token state))
-      (assoc :invitation-token (:invitation-token state))
-
-      ;; If state token comes with props, merge them. The state token
-      ;; props can contain pm_ and utm_ prefixed query params.
-      (map? (:props state))
-      (update :props merge (:props state)))))
-
-;; --- HTTP HANDLERS
-
-(defn extract-props
-  [params]
-  (reduce-kv (fn [params k v]
-               (let [sk (name k)]
-                 (cond-> params
-                   (or (str/starts-with? sk "pm_")
-                       (str/starts-with? sk "pm-")
-                       (str/starts-with? sk "utm_"))
-                   (assoc (-> sk str/kebab keyword) v))))
-             {}
-             params))
-
-(defn- retrieve-profile
-  [{:keys [pool] :as cfg} info]
-  (with-open [conn (db/open pool)]
-    (some->> (:email info)
-             (profile/retrieve-profile-data-by-email conn)
-             (profile/populate-additional-data conn)
-             (profile/decode-profile-row))))
-
-(defn- redirect-response
-  [uri]
-  {:status 302
-   :headers {"location" (str uri)}
-   :body ""})
-
-(defn- generate-error-redirect
-  [cfg error]
-  (let [uri (-> (u/uri (:public-uri cfg))
-                (assoc :path "/#/auth/login")
-                (assoc :query (u/map->query-string {:error "unable-to-auth" :hint (ex-message error)})))]
-    (redirect-response uri)))
-
-(defn- generate-redirect
-  [{:keys [tokens session audit] :as cfg} request info profile]
-  (if profile
-    (let [sxf    ((:create session) (:id profile))
-          token  (or (:invitation-token info)
-                     (tokens :generate {:iss :auth
-                                        :exp (dt/in-future "15m")
-                                        :profile-id (:id profile)}))
-          params {:token token}
-
-          uri    (-> (u/uri (:public-uri cfg))
-                     (assoc :path "/#/auth/verify-token")
-                     (assoc :query (u/map->query-string params)))]
-
-      (when (fn? audit)
-        (audit :cmd :submit
-               :type "mutation"
-               :name "login"
-               :profile-id (:id profile)
-               :ip-addr (audit/parse-client-ip request)
-               :props (audit/profile->props profile)))
-
-      (->> (redirect-response uri)
-           (sxf request)))
-    (let [info   (assoc info
-                        :iss :prepared-register
-                        :exp (dt/in-future {:hours 48}))
-          token  (tokens :generate info)
-          params (d/without-nils
-                  {:token token
-                   :fullname (:fullname info)})
-          uri    (-> (u/uri (:public-uri cfg))
-                     (assoc :path "/#/auth/register/validate")
-                     (assoc :query (u/map->query-string params)))]
-      (redirect-response uri))))
-
-(defn- auth-handler
-  [{:keys [tokens] :as cfg} {:keys [params] :as request}]
-  (let [invitation (:invitation-token params)
-        props      (extract-props params)
-        state      (tokens :generate
-                           {:iss :oauth
-                            :invitation-token invitation
-                            :props props
-                            :exp (dt/in-future "15m")})
-        uri        (build-auth-uri cfg state)]
-    {:status 200
-     :body {:redirect-uri uri}}))
-
-(defn- callback-handler
-  [cfg request]
-  (try
-    (let [info     (retrieve-info cfg request)
-          profile  (retrieve-profile cfg info)]
-      (generate-redirect cfg request info profile))
-    (catch Exception e
-      (l/warn :hint "error on oauth process"
-              :cause e)
-      (generate-error-redirect cfg e))))
-
-;; --- INIT
-
-(declare initialize)
-
-(s/def ::public-uri ::us/not-empty-string)
-(s/def ::session map?)
-(s/def ::tokens fn?)
-(s/def ::rpc map?)
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::public-uri ::session ::tokens ::rpc ::db/pool]))
-
-(defn wrap-handler
-  [cfg handler]
-  (fn [request]
-    (let [provider (get-in request [:path-params :provider])
-          provider (get-in @cfg [:providers provider])]
-      (when-not provider
-        (ex/raise :type :not-found
-                  :context {:provider provider}
-                  :hint "provider not configured"))
-      (-> (assoc @cfg :provider provider)
-          (handler request)))))
-
-(defmethod ig/init-key ::handler
-  [_ cfg]
-  (let [cfg (initialize cfg)]
-    {:handler (wrap-handler cfg auth-handler)
-     :callback-handler (wrap-handler cfg callback-handler)}))
-
-(defn- discover-oidc-config
-  [{:keys [base-uri] :as opts}]
-  (let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
-        response      (http/send! {:method :get :uri (str discovery-uri)})]
-    (when (= 200 (:status response))
-      (let [data (json/read-str (:body response))]
-        (assoc opts
-               :token-uri (get data "token_endpoint")
-               :auth-uri (get data "authorization_endpoint")
-               :user-uri (get data "userinfo_endpoint"))))))
-
-(defn- obfuscate-string
-  [s]
-  (if (< (count s) 10)
-    (apply str (take (count s) (repeat "*")))
-    (str (subs s 0 5)
-         (apply str (take (- (count s) 5) (repeat "*"))))))
-
-(defn- initialize-oidc-provider
-  [cfg]
-  (let [opts {:base-uri      (cf/get :oidc-base-uri)
-              :client-id     (cf/get :oidc-client-id)
-              :client-secret (cf/get :oidc-client-secret)
-              :token-uri     (cf/get :oidc-token-uri)
-              :auth-uri      (cf/get :oidc-auth-uri)
-              :user-uri      (cf/get :oidc-user-uri)
-              :scopes        (cf/get :oidc-scopes #{"openid" "profile" "email"})
-              :roles-attr    (cf/get :oidc-roles-attr)
-              :roles         (cf/get :oidc-roles)
-              :name          "oidc"}]
-    (if (and (string? (:base-uri opts))
-             (string? (:client-id opts))
-             (string? (:client-secret opts)))
-      (if (and (string? (:token-uri opts))
-               (string? (:user-uri opts))
-               (string? (:auth-uri opts)))
-        (do
-          (l/info :action "initialize" :provider "oidc" :method "static"
-                  :opts (pr-str (update opts :client-secret obfuscate-string)))
-          (assoc-in cfg [:providers "oidc"] opts))
-        (let [opts (discover-oidc-config opts)]
-          (l/info :action "initialize" :provider "oidc" :method "discover"
-                  :opts (pr-str (update opts :client-secret obfuscate-string)))
-          (assoc-in cfg [:providers "oidc"] opts)))
-      cfg)))
-
-(defn- initialize-google-provider
-  [cfg]
-  (let [opts {:client-id     (cf/get :google-client-id)
-              :client-secret (cf/get :google-client-secret)
-              :scopes        #{"openid" "email" "profile"}
-              :auth-uri      "https://accounts.google.com/o/oauth2/v2/auth"
-              :token-uri     "https://oauth2.googleapis.com/token"
-              :user-uri      "https://openidconnect.googleapis.com/v1/userinfo"
-              :name          "google"}]
-    (if (and (string? (:client-id opts))
-             (string? (:client-secret opts)))
-      (do
-        (l/info :action "initialize" :provider "google"
-                :opts (pr-str (update opts :client-secret obfuscate-string)))
-        (assoc-in cfg [:providers "google"] opts))
-      cfg)))
-
-(defn- initialize-github-provider
-  [cfg]
-  (let [opts {:client-id     (cf/get :github-client-id)
-              :client-secret (cf/get :github-client-secret)
-              :scopes        #{"read:user" "user:email"}
-              :auth-uri      "https://github.com/login/oauth/authorize"
-              :token-uri     "https://github.com/login/oauth/access_token"
-              :user-uri      "https://api.github.com/user"
-              :name          "github"}]
-    (if (and (string? (:client-id opts))
-             (string? (:client-secret opts)))
-      (do
-        (l/info :action "initialize" :provider "github"
-                :opts (pr-str (update opts :client-secret obfuscate-string)))
-        (assoc-in cfg [:providers "github"] opts))
-      cfg)))
-
-
-(defn- initialize-gitlab-provider
-  [cfg]
-  (let [base (cf/get :gitlab-base-uri "https://gitlab.com")
-        opts {:base-uri      base
-              :client-id     (cf/get :gitlab-client-id)
-              :client-secret (cf/get :gitlab-client-secret)
-              :scopes        #{"read_user"}
-              :auth-uri      (str base "/oauth/authorize")
-              :token-uri     (str base "/oauth/token")
-              :user-uri      (str base "/api/v4/user")
-              :name          "gitlab"}]
-    (if (and (string? (:client-id opts))
-             (string? (:client-secret opts)))
-      (do
-        (l/info :action "initialize" :provider "gitlab"
-                :opts (pr-str (update opts :client-secret obfuscate-string)))
-        (assoc-in cfg [:providers "gitlab"] opts))
-      cfg)))
-
-(defn- initialize
-  [cfg]
-  (let [cfg (agent cfg :error-mode :continue)]
-    (send-off cfg initialize-google-provider)
-    (send-off cfg initialize-gitlab-provider)
-    (send-off cfg initialize-github-provider)
-    (send-off cfg initialize-oidc-provider)
-    cfg))

backend/src/app/http/session.clj

@@ -7,158 +7,273 @@
 (ns app.http.session
   (:require
    [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
+   [app.common.logging :as l]
+   [app.config :as cf]
    [app.db :as db]
-   [app.metrics :as mtx]
-   [app.util.async :as aa]
-   [app.util.logging :as l]
+   [app.db.sql :as sql]
    [app.util.time :as dt]
    [app.worker :as wrk]
-   [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.request :as yrq]))
 
-;; A default cookie name for storing the session. We don't allow
-;; configure it.
-(def cookie-name "auth-token")
+;; A default cookie name for storing the session.
+(def default-auth-token-cookie-name "auth-token")
+
+;; A cookie that we can use to check from other sites of the same
+;; domain if a user is authenticated.
+(def default-authenticated-cookie-name "authenticated")
+
+;; Default value for cookie max-age
+(def default-cookie-max-age (dt/duration {:days 7}))
+
+;; Default age for automatic session renewal
+(def default-renewal-max-age (dt/duration {:hours 6}))
+
+(defprotocol ISessionStore
+  (read-session [store key])
+  (write-session [store key data])
+  (update-session [store data])
+  (delete-session [store key]))
+
+(defn- make-database-store
+  [{:keys [pool tokens executor]}]
+  (reify ISessionStore
+    (read-session [_ token]
+      (px/with-dispatch executor
+        (db/exec-one! pool (sql/select :http-session {:id token}))))
+
+    (write-session [_ _ data]
+      (px/with-dispatch executor
+        (let [profile-id (:profile-id data)
+              user-agent (:user-agent data)
+              created-at (or (:created-at data) (dt/now))
+              token      (tokens :generate {:iss "authentication"
+                                            :iat created-at
+                                            :uid profile-id})
+              params     {:user-agent user-agent
+                          :profile-id profile-id
+                          :created-at created-at
+                          :updated-at created-at
+                          :id token}]
+          (db/insert! pool :http-session params))))
+
+    (update-session [_ data]
+      (let [updated-at (dt/now)]
+        (px/with-dispatch executor
+          (db/update! pool :http-session
+                      {:updated-at updated-at}
+                      {:id (:id data)})
+          (assoc data :updated-at updated-at))))
+
+
+    (delete-session [_ token]
+      (px/with-dispatch executor
+        (db/delete! pool :http-session {:id token})
+        nil))))
+
+(defn make-inmemory-store
+  [{:keys [tokens]}]
+  (let [cache (atom {})]
+    (reify ISessionStore
+      (read-session [_ token]
+        (p/do (get @cache token)))
+
+      (write-session [_ _ data]
+        (p/do
+          (let [profile-id (:profile-id data)
+                user-agent (:user-agent data)
+                created-at (or (:created-at data) (dt/now))
+                token      (tokens :generate {:iss "authentication"
+                                              :iat created-at
+                                              :uid profile-id})
+                params     {:user-agent user-agent
+                            :created-at created-at
+                            :updated-at created-at
+                            :profile-id profile-id
+                            :id token}]
+
+            (swap! cache assoc token params)
+            params)))
+
+      (update-session [_ data]
+        (let [updated-at (dt/now)]
+          (swap! cache update (:id data) assoc :updated-at updated-at)
+          (assoc data :updated-at updated-at)))
+
+      (delete-session [_ token]
+        (p/do
+          (swap! cache dissoc token)
+          nil)))))
+
+(s/def ::tokens fn?)
+(defmethod ig/pre-init-spec ::store [_]
+  (s/keys :req-un [::db/pool ::wrk/executor ::tokens]))
+
+(defmethod ig/init-key ::store
+  [_ {:keys [pool] :as cfg}]
+  (if (db/read-only? pool)
+    (make-inmemory-store cfg)
+    (make-database-store cfg)))
+
+(defmethod ig/halt-key! ::store
+  [_ _])
 
 ;; --- IMPL
 
-(defn- create-session
-  [{:keys [conn tokens] :as cfg} {:keys [profile-id headers] :as request}]
-  (let [token  (tokens :generate {:iss "authentication"
-                                  :iat (dt/now)
-                                  :uid profile-id})
-        params {:user-agent (get headers "user-agent")
-                :profile-id profile-id
-                :id token}]
-    (db/insert! conn :http-session params)))
+(defn- create-session!
+  [store profile-id user-agent]
+  (let [params {:user-agent user-agent
+                :profile-id profile-id}]
+    (write-session store nil params)))
 
-(defn- delete-session
-  [{:keys [conn] :as cfg} {:keys [cookies] :as request}]
-  (when-let [token (get-in cookies [cookie-name :value])]
-    (db/delete! conn :http-session {:id token}))
-  nil)
+(defn- update-session!
+  [store session]
+  (update-session store session))
+
+(defn- delete-session!
+  [store {:keys [cookies] :as request}]
+  (let [name (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
+    (when-let [token (get-in cookies [name :value])]
+      (delete-session store token))))
 
 (defn- retrieve-session
-  [{:keys [conn] :as cfg} id]
-  (when id
-    (db/exec-one! conn ["select id, profile_id from http_session where id = ?" id])))
+  [store request]
+  (let [cookie-name (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
+    (when-let [cookie (yrq/get-cookie request cookie-name)]
+      (read-session store (:value cookie)))))
 
-(defn- retrieve-from-request
-  [cfg {:keys [cookies] :as request}]
-  (->> (get-in cookies [cookie-name :value])
-       (retrieve-session cfg)))
+(defn assign-auth-token-cookie
+  [response {token :id updated-at :updated-at}]
+  (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
+        created-at (or updated-at (dt/now))
+        renewal    (dt/plus created-at default-renewal-max-age)
+        expires    (dt/plus created-at max-age)
+        secure?    (contains? cf/flags :secure-session-cookies)
+        cors?      (contains? cf/flags :cors)
+        name       (cf/get :auth-token-cookie-name default-auth-token-cookie-name)
+        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
+        cookie     {:path "/"
+                    :http-only true
+                    :expires expires
+                    :value token
+                    :comment comment
+                    :same-site (if cors? :none :lax)
+                    :secure secure?}]
+    (update response :cookies assoc name cookie)))
 
-(defn- add-cookies
-  [response {:keys [id] :as session}]
-  (assoc response :cookies {cookie-name {:path "/" :http-only true :value id}}))
+(defn assign-authenticated-cookie
+  [response {updated-at :updated-at}]
+  (let [max-age    (cf/get :auth-token-cookie-max-age default-cookie-max-age)
+        created-at (or updated-at (dt/now))
+        renewal    (dt/plus created-at default-renewal-max-age)
+        expires    (dt/plus created-at max-age)
+        comment    (str "Renewal at: " (dt/format-instant renewal :rfc1123))
+        secure?    (contains? cf/flags :secure-session-cookies)
+        domain     (cf/get :authenticated-cookie-domain)
+        name       (cf/get :authenticated-cookie-name "authenticated")
+        cookie     {:domain domain
+                    :expires expires
+                    :path "/"
+                    :comment comment
+                    :value true
+                    :same-site :strict
+                    :secure secure?}]
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc name cookie))))
 
-(defn- clear-cookies
+(defn clear-auth-token-cookie
   [response]
-  (assoc response :cookies {cookie-name {:value "" :max-age -1}}))
+  (let [name (cf/get :auth-token-cookie-name default-auth-token-cookie-name)]
+    (update response :cookies assoc name {:path "/" :value "" :max-age -1})))
+
+(defn- clear-authenticated-cookie
+  [response]
+  (let [name   (cf/get :authenticated-cookie-name default-authenticated-cookie-name)
+        domain (cf/get :authenticated-cookie-domain)]
+    (cond-> response
+      (string? domain)
+      (update :cookies assoc name {:domain domain :path "/" :value "" :max-age -1}))))
+
+(defn- make-middleware
+  [{:keys [store] :as cfg}]
+  (letfn [;; Check if time reached for automatic session renewal
+          (renew-session? [{:keys [updated-at] :as session}]
+            (and (dt/instant? updated-at)
+                 (let [elapsed (dt/diff updated-at (dt/now))]
+                   (neg? (compare default-renewal-max-age elapsed)))))
+
+          ;; Wrap respond with session renewal code
+          (wrap-respond [respond session]
+            (fn [response]
+              (p/let [session (update-session! store session)]
+                (-> response
+                    (assign-auth-token-cookie session)
+                    (assign-authenticated-cookie session)
+                    (respond)))))]
+
+    {:name :session
+     :compile (fn [& _]
+                (fn [handler]
+                  (fn [request respond raise]
+                    (try
+                      (-> (retrieve-session store request)
+                          (p/finally (fn [session cause]
+                                       (cond
+                                         (some? cause)
+                                         (raise cause)
+
+                                         (nil? session)
+                                         (handler request respond raise)
+
+                                         :else
+                                         (let [request    (-> request
+                                                              (assoc :profile-id (:profile-id session))
+                                                              (assoc :session-id (:id session)))
+                                               respond    (cond-> respond
+                                                            (renew-session? session)
+                                                            (wrap-respond session))]
+
+                                           (handler request respond raise))))))
+
+                      (catch Throwable cause
+                        (raise cause))))))}))
 
-(defn- middleware
-  [cfg handler]
-  (fn [request]
-    (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
-      (do
-        (a/>!! (::events-ch cfg) id)
-        (l/update-thread-context! {:profile-id profile-id})
-        (handler (assoc request :profile-id profile-id)))
-      (handler request))))
 
 ;; --- STATE INIT: SESSION
 
-(defmethod ig/pre-init-spec ::session [_]
-  (s/keys :req-un [::db/pool]))
+(s/def ::store #(satisfies? ISessionStore %))
 
-(defmethod ig/prep-key ::session
+(defmethod ig/pre-init-spec :app.http/session [_]
+  (s/keys :req-un [::store]))
+
+(defmethod ig/prep-key :app.http/session
   [_ cfg]
-  (d/merge {:buffer-size 64}
+  (d/merge {:buffer-size 128}
            (d/without-nils cfg)))
 
-(defmethod ig/init-key ::session
-  [_ {:keys [pool] :as cfg}]
-  (let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
-        cfg    (-> cfg
-                   (assoc :conn pool)
-                   (assoc ::events-ch events))]
-    (-> cfg
-        (assoc :middleware #(middleware cfg %))
-        (assoc :create (fn [profile-id]
-                         (fn [request response]
-                           (let [request (assoc request :profile-id profile-id)
-                                 session (create-session cfg request)]
-                             (add-cookies response session)))))
-        (assoc :delete (fn [request response]
-                         (delete-session cfg request)
+(defmethod ig/init-key :app.http/session
+  [_ {:keys [store] :as cfg}]
+  (-> cfg
+      (assoc :middleware (make-middleware cfg))
+      (assoc :create (fn [profile-id]
+                       (fn [request response]
+                         (p/let [uagent  (yrq/get-header request "user-agent")
+                                 session (create-session! store profile-id uagent)]
+                           (-> response
+                               (assign-auth-token-cookie session)
+                               (assign-authenticated-cookie session))))))
+      (assoc :delete (fn [request response]
+                       (p/do
+                         (delete-session! store request)
                          (-> response
                              (assoc :status 204)
-                             (assoc :body "")
-                             (clear-cookies)))))))
-
-(defmethod ig/halt-key! ::session
-  [_ data]
-  (a/close! (::events-ch data)))
-
-
-;; --- STATE INIT: SESSION UPDATER
-
-(declare update-sessions)
-
-(s/def ::session map?)
-(s/def ::max-batch-age ::cfg/http-session-updater-batch-max-age)
-(s/def ::max-batch-size ::cfg/http-session-updater-batch-max-size)
-
-(defmethod ig/pre-init-spec ::updater [_]
-  (s/keys :req-un [::db/pool ::wrk/executor ::mtx/metrics ::session]
-          :opt-un [::max-batch-age
-                   ::max-batch-size]))
-
-(defmethod ig/prep-key ::updater
-  [_ cfg]
-  (merge {:max-batch-age (dt/duration {:minutes 5})
-          :max-batch-size 200}
-         (d/without-nils cfg)))
-
-(defmethod ig/init-key ::updater
-  [_ {:keys [session metrics] :as cfg}]
-  (l/info :action "initialize session updater"
-          :max-batch-age (str (:max-batch-age cfg))
-          :max-batch-size (str (:max-batch-size cfg)))
-  (let [input (aa/batch (::events-ch session)
-                        {:max-batch-size (:max-batch-size cfg)
-                         :max-batch-age (inst-ms (:max-batch-age cfg))})
-        mcnt  (mtx/create
-               {:name "http_session_update_total"
-                :help "A counter of session update batch events."
-                :registry (:registry metrics)
-                :type :counter})]
-    (a/go-loop []
-      (when-let [[reason batch] (a/<! input)]
-        (let [result (a/<! (update-sessions cfg batch))]
-          (mcnt :inc)
-          (cond
-            (ex/exception? result)
-            (l/error :task "updater"
-                     :hint "unexpected error on update sessions"
-                     :cause result)
-
-            (= :size reason)
-            (l/debug :task "updater"
-                     :action "update sessions"
-                     :reason (name reason)
-                     :count result))
-          (recur))))))
-
-(defn- update-sessions
-  [{:keys [pool executor]} ids]
-  (aa/with-thread executor
-    (db/exec-one! pool ["update http_session set updated_at=now() where id = ANY(?)"
-                        (into-array String ids)])
-    (count ids)))
+                             (assoc :body nil)
+                             (clear-auth-token-cookie)
+                             (clear-authenticated-cookie)))))))
 
 ;; --- STATE INIT: SESSION GC
 
@@ -172,22 +287,25 @@
 
 (defmethod ig/prep-key ::gc-task
   [_ cfg]
-  (merge {:max-age (dt/duration {:days 2})}
+  (merge {:max-age default-cookie-max-age}
          (d/without-nils cfg)))
 
 (defmethod ig/init-key ::gc-task
   [_ {:keys [pool max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
   (fn [_]
     (db/with-atomic [conn pool]
       (let [interval (db/interval max-age)
-            result   (db/exec-one! conn [sql:delete-expired interval])
+            result   (db/exec-one! conn [sql:delete-expired interval interval])
             result   (:next.jdbc/update-count result)]
         (l/debug :task "gc"
-                 :action "clean http sessions"
-                 :count result)
+                 :hint "clean http sessions"
+                 :deleted result)
         result))))
 
 (def ^:private
   sql:delete-expired
   "delete from http_session
-    where updated_at < now() - ?::interval")
+    where updated_at < now() - ?::interval
+       or (updated_at is null and
+           created_at < now() - ?::interval)")

backend/src/app/http/websocket.clj

@@ -0,0 +1,341 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.websocket
+  "A penpot notification service for file cooperative edition."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.pprint :as pp]
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.metrics :as mtx]
+   [app.util.time :as dt]
+   [app.util.websocket :as ws]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [yetti.websocket :as yws]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; WEBSOCKET HOOKS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def state (atom {}))
+
+(defn- on-connect
+  [{:keys [metrics]} wsp]
+  (let [created-at (dt/now)]
+    (swap! state assoc (::ws/id @wsp) wsp)
+    (mtx/run! metrics {:id :websocket-active-connections :inc 1})
+    (fn []
+      (swap! state dissoc (::ws/id @wsp))
+      (mtx/run! metrics {:id :websocket-active-connections :dec 1})
+      (mtx/run! metrics {:id :websocket-session-timing
+                         :val (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0)}))))
+
+(defn- on-rcv-message
+  [{:keys [metrics]} _ message]
+  (mtx/run! metrics {:id :websocket-messages-total :labels ["recv"] :inc 1})
+  message)
+
+(defn- on-snd-message
+  [{:keys [metrics]} _ message]
+  (mtx/run! metrics {:id :websocket-messages-total :labels ["send"] :inc 1})
+  message)
+
+;; REPL HELPERS
+
+(defn repl-get-connections-for-file
+  [file-id]
+  (->> (vals @state)
+       (filter #(= file-id (-> % deref ::file-subscription :file-id)))
+       (map deref)
+       (map ::ws/id)))
+
+(defn repl-get-connections-for-team
+  [team-id]
+  (->> (vals @state)
+       (filter #(= team-id (-> % deref ::team-subscription :team-id)))
+       (map deref)
+       (map ::ws/id)))
+
+(defn repl-close-connection
+  [id]
+  (when-let [wsp (get @state id)]
+    (a/>!! (::ws/close-ch @wsp) [8899 "closed from server"])
+    (a/close! (::ws/close-ch @wsp))))
+
+(defn repl-get-connection-info
+  [id]
+  (when-let [wsp (get @state id)]
+    {:id id
+     :created-at (dt/instant id)
+     :profile-id (::profile-id @wsp)
+     :session-id (::session-id @wsp)
+     :user-agent (::ws/user-agent @wsp)
+     :ip-addr    (::ws/remote-addr @wsp)
+     :last-activity-at (::ws/last-activity-at @wsp)
+     :http-session-id  (::ws/http-session-id @wsp)
+     :subscribed-file  (-> wsp deref ::file-subscription :file-id)
+     :subscribed-team  (-> wsp deref ::team-subscription :team-id)}))
+
+(defn repl-print-connection-info
+  [id]
+  (some-> id repl-get-connection-info pp/pprint))
+
+(defn repl-print-connection-info-for-file
+  [file-id]
+  (some->> (repl-get-connections-for-file file-id)
+           (map repl-get-connection-info)
+           (pp/pprint)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; WEBSOCKET HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmulti handle-message
+  (fn [_ _ message]
+    (:type message)))
+
+(defmethod handle-message :connect
+  [cfg wsp _]
+
+  (let [msgbus-fn  (:msgbus cfg)
+        conn-id    (::ws/id @wsp)
+        profile-id (::profile-id @wsp)
+        session-id (::session-id @wsp)
+        output-ch  (::ws/output-ch @wsp)
+
+        xform      (remove #(= (:session-id %) session-id))
+        channel    (a/chan (a/dropping-buffer 16) xform)]
+
+    (l/trace :fn "handle-message" :event :connect :conn-id conn-id)
+
+    ;; Subscribe to the profile channel and forward all messages to
+    ;; websocket output channel (send them to the client).
+    (swap! wsp assoc ::profile-subscription channel)
+    (a/pipe channel output-ch false)
+    (msgbus-fn :cmd :sub :topic profile-id :chan channel)))
+
+(defmethod handle-message :disconnect
+  [cfg wsp _]
+  (let [msgbus-fn  (:msgbus cfg)
+        conn-id    (::ws/id @wsp)
+        profile-id (::profile-id @wsp)
+        session-id (::session-id @wsp)
+        profile-ch (::profile-subscription @wsp)
+        fsub       (::file-subscription @wsp)
+        tsub       (::team-subscription @wsp)
+
+        message    {:type :disconnect
+                    :subs-id profile-id
+                    :profile-id profile-id
+                    :session-id session-id}]
+
+    (l/trace :fn "handle-message"
+             :event :disconnect
+             :conn-id conn-id)
+
+    (a/go
+      ;; Close the main profile subscription
+      (a/close! profile-ch)
+      (a/<! (msgbus-fn :cmd :purge :chans [profile-ch]))
+
+      ;; Close tram subscription if exists
+      (when-let [channel (:channel tsub)]
+        (a/close! channel)
+        (a/<! (msgbus-fn :cmd :purge :chans [channel])))
+
+      (when-let [{:keys [topic channel]} fsub]
+        (a/close! channel)
+        (a/<! (msgbus-fn :cmd :purge :chans [channel]))
+        (a/<! (msgbus-fn :cmd :pub :topic topic :message message))))))
+
+(defmethod handle-message :subscribe-team
+  [cfg wsp {:keys [team-id] :as params}]
+  (let [msgbus-fn  (:msgbus cfg)
+        conn-id    (::ws/id @wsp)
+        session-id (::session-id @wsp)
+        output-ch  (::ws/output-ch @wsp)
+        prev-subs  (get @wsp ::team-subscription)
+        xform      (comp
+                    (remove #(= (:session-id %) session-id))
+                    (map #(assoc % :subs-id team-id)))
+
+        channel    (a/chan (a/dropping-buffer 64) xform)]
+
+    (l/trace :fn "handle-message"
+             :event :subscribe-team
+             :team-id team-id
+             :conn-id conn-id)
+
+    (a/pipe channel output-ch false)
+
+    (let [state {:team-id team-id :channel channel :topic team-id}]
+      (swap! wsp assoc ::team-subscription state))
+
+    (a/go
+      ;; Close previous subscription if exists
+      (when-let [channel (:channel prev-subs)]
+        (a/close! channel)
+        (a/<! (msgbus-fn :cmd :purge :chans [channel]))))
+
+    (a/go
+      (a/<! (msgbus-fn :cmd :sub :topic team-id :chan channel)))))
+
+(defmethod handle-message :subscribe-file
+  [cfg wsp {:keys [file-id] :as params}]
+  (let [msgbus-fn  (:msgbus cfg)
+        conn-id    (::ws/id @wsp)
+        profile-id (::profile-id @wsp)
+        session-id (::session-id @wsp)
+        output-ch  (::ws/output-ch @wsp)
+        prev-subs  (::file-subscription @wsp)
+        xform      (comp (remove #(= (:session-id %) session-id))
+                         (map #(assoc % :subs-id file-id)))
+        channel    (a/chan (a/dropping-buffer 64) xform)]
+
+    (l/trace :fn "handle-message"
+             :event :subscribe-file
+             :file-id file-id
+             :conn-id conn-id)
+
+    (let [state {:file-id file-id :channel channel :topic file-id}]
+      (swap! wsp assoc ::file-subscription state))
+
+    (a/go
+      ;; Close previous subscription if exists
+      (when-let [channel (:channel prev-subs)]
+        (a/close! channel)
+        (a/<! (msgbus-fn :cmd :purge :chans [channel]))))
+
+    ;; Message forwarding
+    (a/go
+      (loop []
+        (when-let [{:keys [type] :as message} (a/<! channel)]
+          (when (or (= :join-file type)
+                    (= :leave-file type)
+                    (= :disconnect type))
+            (let [message {:type :presence
+                           :file-id file-id
+                           :session-id session-id
+                           :profile-id profile-id}]
+              (a/<! (msgbus-fn :cmd :pub
+                               :topic file-id
+                               :message message))))
+          (a/>! output-ch message)
+          (recur))))
+
+    (a/go
+      ;; Subscribe to file topic
+      (a/<! (msgbus-fn :cmd :sub :topic file-id :chan channel))
+
+      ;; Notifify the rest of participants of the new connection.
+      (let [message {:type :join-file
+                     :file-id file-id
+                     :subs-id file-id
+                     :session-id session-id
+                     :profile-id profile-id}]
+        (a/<! (msgbus-fn :cmd :pub
+                         :topic file-id
+                         :message message))))))
+
+(defmethod handle-message :unsubscribe-file
+  [cfg wsp {:keys [file-id] :as params}]
+  (let [msgbus-fn  (:msgbus cfg)
+        conn-id    (::ws/id @wsp)
+        session-id (::session-id @wsp)
+        profile-id (::profile-id @wsp)
+        subs       (::file-subscription @wsp)
+
+        message    {:type :leave-file
+                    :file-id file-id
+                    :session-id session-id
+                    :profile-id profile-id}]
+
+    (l/trace :fn "handle-message"
+             :event :unsubscribe-file
+             :file-id file-id
+             :conn-id conn-id)
+
+    (a/go
+      (when (= (:file-id subs) file-id)
+        (let [channel (:channel subs)]
+          (a/close! channel)
+          (a/<! (msgbus-fn :cmd :purge :chans [channel]))
+          (a/<! (msgbus-fn :cmd :pub :topic file-id :message message)))))))
+
+(defmethod handle-message :keepalive
+  [_ _ _]
+  (l/trace :fn "handle-message" :event :keepalive)
+  (a/go :nothing))
+
+(defmethod handle-message :pointer-update
+  [cfg wsp {:keys [file-id] :as message}]
+  (let [msgbus-fn  (:msgbus cfg)
+        profile-id (::profile-id @wsp)
+        session-id (::session-id @wsp)
+        subs       (::file-subscription @wsp)
+        message    (-> message
+                       (assoc :subs-id file-id)
+                       (assoc :profile-id profile-id)
+                       (assoc :session-id session-id))]
+    (a/go
+      ;; Only allow receive pointer updates when active subscription
+      (when subs
+        (a/<! (msgbus-fn :cmd :pub
+                         :topic file-id
+                         :message message))))))
+
+(defmethod handle-message :default
+  [_ wsp message]
+  (let [conn-id (::ws/id @wsp)]
+    (l/warn :hint "received unexpected message"
+            :message message
+            :conn-id conn-id)
+    (a/go :none)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP HANDLER
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::msgbus fn?)
+(s/def ::session-id ::us/uuid)
+
+(s/def ::handler-params
+  (s/keys :req-un [::session-id]))
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::msgbus ::db/pool ::mtx/metrics]))
+
+(defmethod ig/init-key ::handler
+  [_ cfg]
+  (fn [{:keys [profile-id params] :as req} respond raise]
+    (let [{:keys [session-id]} (us/conform ::handler-params params)]
+      (cond
+        (not profile-id)
+        (raise (ex/error :type :authentication
+                         :hint "Authentication required."))
+
+        (not (yws/upgrade-request? req))
+        (raise (ex/error :type :validation
+                         :code :websocket-request-expected
+                         :hint "this endpoint only accepts websocket connections"))
+
+        :else
+        (do
+          (l/trace :hint "websocket request" :profile-id profile-id :session-id session-id)
+
+          (->> (ws/handler
+                ::ws/on-rcv-message (partial on-rcv-message cfg)
+                ::ws/on-snd-message (partial on-snd-message cfg)
+                ::ws/on-connect (partial on-connect cfg)
+                ::ws/handler (partial handle-message cfg)
+                ::profile-id profile-id
+                ::session-id session-id)
+               (yws/upgrade req)
+               (respond)))))))

backend/src/app/loggers/audit.clj

@@ -9,63 +9,143 @@
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.util.async :as aa]
-   [app.util.http :as http]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [integrant.core :as ig]
-   [lambdaisland.uri :as u]))
+   [lambdaisland.uri :as u]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.request :as yrq]
+   [yetti.response :as yrs]))
 
 (defn parse-client-ip
-  [{:keys [headers] :as request}]
-  (or (some-> (get headers "x-forwarded-for") (str/split ",") first)
-      (get headers "x-real-ip")
-      (get request :remote-addr)))
+  [request]
+  (or (some-> (yrq/get-header request "x-forwarded-for") (str/split ",") first)
+      (yrq/get-header request "x-real-ip")
+      (some-> (yrq/remote-addr request) str)))
+
+(defn extract-utm-params
+  "Extracts additional data from params and namespace them under
+  `penpot` ns."
+  [params]
+  (letfn [(process-param [params k v]
+            (let [sk (d/name k)]
+              (cond-> params
+                (str/starts-with? sk "utm_")
+                (assoc (->> sk str/kebab (keyword "penpot")) v)
+
+                (str/starts-with? sk "mtm_")
+                (assoc (->> sk str/kebab (keyword "penpot")) v))))]
+    (reduce-kv process-param {} params)))
 
 (defn profile->props
   [profile]
   (-> profile
-      (select-keys [:is-active :is-muted :auth-backend :email :default-team-id :default-project-id :fullname :lang])
+      (select-keys [:id :is-active :is-muted :auth-backend :email :default-team-id :default-project-id :fullname :lang])
+      (merge (:props profile))
       (d/without-nils)))
 
 (defn clean-props
   [{:keys [profile-id] :as event}]
-  (letfn [(clean-common [props]
-            (-> props
-                (dissoc :session-id)
-                (dissoc :password)
-                (dissoc :old-password)
-                (dissoc :token)))
+  (let [invalid-keys #{:session-id
+                       :password
+                       :old-password
+                       :token}
+        xform (comp
+               (remove (fn [kv]
+                         (qualified-keyword? (first kv))))
+               (remove (fn [kv]
+                         (contains? invalid-keys (first kv))))
+               (remove (fn [[k v]]
+                         (and (= k :profile-id)
+                              (= v profile-id))))
+               (filter (fn [[_ v]]
+                         (or (string? v)
+                             (keyword? v)
+                             (uuid? v)
+                             (boolean? v)
+                             (number? v)))))]
 
-          (clean-profile-id [props]
-            (cond-> props
-              (= profile-id (:profile-id props))
-              (dissoc :profile-id)))
+    (update event :props #(into {} xform %))))
 
-          (clean-complex-data [props]
-            (reduce-kv (fn [props k v]
-                         (cond-> props
-                           (or (string? v)
-                               (uuid? v)
-                               (boolean? v)
-                               (number? v))
-                           (assoc k v)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HTTP Handler
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-                           (keyword? v)
-                           (assoc k (name v))))
-                       {}
-                       props))]
+(declare persist-http-events)
 
-    (update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
+(s/def ::profile-id ::us/uuid)
+(s/def ::name ::us/string)
+(s/def ::type ::us/string)
+(s/def ::props (s/map-of ::us/keyword any?))
+(s/def ::timestamp dt/instant?)
+(s/def ::context (s/map-of ::us/keyword any?))
+
+(s/def ::frontend-event
+  (s/keys :req-un [::type ::name ::props ::timestamp ::profile-id]
+          :opt-un [::context]))
+
+(s/def ::frontend-events (s/every ::frontend-event))
+
+(defmethod ig/init-key ::http-handler
+  [_  {:keys [executor pool] :as cfg}]
+  (if (or (db/read-only? pool) (not (contains? cf/flags :audit-log)))
+    (do
+      (l/warn :hint "audit log http handler disabled or db is read-only")
+      (fn [_ respond _]
+        (respond (yrs/response 204))))
+
+    (letfn [(handler [{:keys [profile-id] :as request}]
+              (let [events  (->> (:events (:params request))
+                                 (remove #(not= profile-id (:profile-id %)))
+                                 (us/conform ::frontend-events))
+
+                    ip-addr (parse-client-ip request)
+                    cfg     (-> cfg
+                                (assoc :source "frontend")
+                                (assoc :events events)
+                                (assoc :ip-addr ip-addr))]
+                (persist-http-events cfg)))
+
+            (handle-error [cause]
+              (let [xdata (ex-data cause)]
+                (if (= :spec-validation (:code xdata))
+                  (l/error ::l/raw (str "spec validation on persist-events:\n" (us/pretty-explain xdata)))
+                  (l/error :hint "error on persist-events" :cause cause))))]
+
+      (fn [request respond _]
+        ;; Fire and forget, log error in case of errro
+        (-> (px/submit! executor #(handler request))
+            (p/catch handle-error))
+
+        (respond (yrs/response 204))))))
+
+(defn- persist-http-events
+  [{:keys [pool events ip-addr source] :as cfg}]
+  (let [columns    [:id :name :source :type :tracked-at :profile-id :ip-addr :props :context]
+        prepare-xf (map (fn [event]
+                          [(uuid/next)
+                           (:name event)
+                           source
+                           (:type event)
+                           (:timestamp event)
+                           (:profile-id event)
+                           (db/inet ip-addr)
+                           (db/tjson (:props event))
+                           (db/tjson (d/without-nils (:context event)))]))]
+    (when (seq events)
+      (->> (into [] prepare-xf events)
+           (db/insert-multi! pool :audit-log columns)))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Collector
@@ -76,39 +156,57 @@
 ;; an external storage and data cleared.
 
 (declare persist-events)
-(s/def ::enabled ::us/boolean)
 
 (defmethod ig/pre-init-spec ::collector [_]
-  (s/keys :req-un [::db/pool ::wrk/executor ::enabled]))
+  (s/keys :req-un [::db/pool ::wrk/executor]))
 
-(def event-xform
+(s/def ::ip-addr string?)
+(s/def ::backend-event
+  (s/keys :req-un [::type ::name ::profile-id]
+          :opt-un [::ip-addr ::props]))
+
+(def ^:private backend-event-xform
   (comp
-   (filter :profile-id)
+   (filter #(us/valid? ::backend-event %))
    (map clean-props)))
 
 (defmethod ig/init-key ::collector
-  [_ {:keys [enabled] :as cfg}]
-  (when enabled
-    (l/info :msg "intializing audit collector")
-    (let [input  (a/chan 512 event-xform)
+  [_ {:keys [pool] :as cfg}]
+  (cond
+    (not (contains? cf/flags :audit-log))
+    (do
+      (l/info :hint "audit log collection disabled")
+      (constantly nil))
+
+    (db/read-only? pool)
+    (do
+      (l/warn :hint "audit log collection disabled, db is read-only")
+      (constantly nil))
+
+    :else
+    (let [input  (a/chan 512 backend-event-xform)
           buffer (aa/batch input {:max-batch-size 100
                                   :max-batch-age (* 10 1000) ; 10s
                                   :init []})]
+      (l/info :hint "audit log collector initialized")
       (a/go-loop []
         (when-let [[_type events] (a/<! buffer)]
           (let [res (a/<! (persist-events cfg events))]
             (when (ex/exception? res)
-              (l/error :hint "error on persiting events"
-                       :cause res)))
-          (recur)))
+              (l/error :hint "error on persisting events" :cause res))
+            (recur))))
 
       (fn [& {:keys [cmd] :as params}]
-        (let [params (dissoc params :cmd)]
-          (case cmd
-            :stop   (a/close! input)
-            :submit (when-not (a/offer! input params)
-                      (l/warn :msg "activity channel is full"))))))))
+        (case cmd
+          :stop
+          (a/close! input)
 
+          :submit
+          (let [params (-> params
+                           (dissoc :cmd)
+                           (assoc :tracked-at (dt/now)))]
+            (when-not (a/offer! input params)
+              (l/warn :hint "activity channel is full"))))))))
 
 (defn- persist-events
   [{:keys [pool executor] :as cfg} events]
@@ -117,69 +215,89 @@
              (:name event)
              (:type event)
              (:profile-id event)
+             (:tracked-at event)
              (some-> (:ip-addr event) db/inet)
-             (db/tjson (:props event))])]
-
+             (db/tjson (:props event))
+             "backend"])]
     (aa/with-thread executor
-      (db/with-atomic [conn pool]
-        (db/insert-multi! conn :audit-log
-                          [:id :name :type :profile-id :ip-addr :props]
-                          (sequence (map event->row) events))))))
+      (when (seq events)
+        (db/with-atomic [conn pool]
+          (db/insert-multi! conn :audit-log
+                            [:id :name :type :profile-id :tracked-at :ip-addr :props :source]
+                            (sequence (keep event->row) events)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Archive Task
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-;; This is a task responsible to send the accomulated events to an
+;; This is a task responsible to send the accumulated events to an
 ;; external service for archival.
 
 (declare archive-events)
 
+(s/def ::http-client fn?)
 (s/def ::uri ::us/string)
 (s/def ::tokens fn?)
 
 (defmethod ig/pre-init-spec ::archive-task [_]
-  (s/keys :req-un [::db/pool ::tokens ::enabled]
+  (s/keys :req-un [::db/pool ::tokens ::http-client]
           :opt-un [::uri]))
 
 (defmethod ig/init-key ::archive-task
-  [_ {:keys [uri enabled] :as cfg}]
-  (fn [_]
-    (when (and enabled (not uri))
-      (ex/raise :type :internal
-                :code :task-not-configured
-                :hint "archive task not configured, missing uri"))
-    (loop []
-      (let [res (archive-events cfg)]
-        (when (= res :continue)
-          (aa/thread-sleep 200)
-          (recur))))))
+  [_ {:keys [uri] :as cfg}]
+  (fn [props]
+    ;; NOTE: this let allows overwrite default configured values from
+    ;; the repl, when manually invoking the task.
+    (let [enabled (or (contains? cf/flags :audit-log-archive)
+                      (:enabled props false))
+          uri     (or uri (:uri props))
+          cfg     (assoc cfg :uri uri)]
+
+      (when (and enabled (not uri))
+        (ex/raise :type :internal
+                  :code :task-not-configured
+                  :hint "archive task not configured, missing uri"))
+
+      (when enabled
+        (loop [total 0]
+          (let [n (archive-events cfg)]
+            (if n
+              (do
+                (aa/thread-sleep 200)
+                (recur (+ total n)))
+              (when (pos? total)
+                (l/trace :hint "events chunk archived" :num total)))))))))
 
 (def sql:retrieve-batch-of-audit-log
   "select * from audit_log
     where archived_at is null
     order by created_at asc
-    limit 100
+    limit 256
       for update skip locked;")
 
 (defn archive-events
-  [{:keys [pool uri tokens] :as cfg}]
-  (letfn [(decode-row [{:keys [props ip-addr] :as row}]
+  [{:keys [pool uri tokens http-client] :as cfg}]
+  (letfn [(decode-row [{:keys [props ip-addr context] :as row}]
             (cond-> row
               (db/pgobject? props)
               (assoc :props (db/decode-transit-pgobject props))
 
+              (db/pgobject? context)
+              (assoc :context (db/decode-transit-pgobject context))
+
               (db/pgobject? ip-addr "inet")
               (assoc :ip-addr (db/decode-inet ip-addr))))
 
-          (row->event [{:keys [name type created-at profile-id props ip-addr]}]
-            (cond-> {:type type
-                     :name name
-                     :timestamp created-at
-                     :profile-id profile-id
-                     :props props}
-              (some? ip-addr)
-              (update :context assoc :source-ip ip-addr)))
+          (row->event [row]
+            (select-keys row [:type
+                              :name
+                              :source
+                              :created-at
+                              :tracked-at
+                              :profile-id
+                              :ip-addr
+                              :props
+                              :context]))
 
           (send [events]
             (let [token   (tokens :generate {:iss "authentication"
@@ -194,12 +312,14 @@
                            :method :post
                            :headers headers
                            :body body}
-                  resp    (http/send! params)]
-              (when (not= (:status resp) 204)
-                (ex/raise :type :internal
-                          :code :unable-to-send-events
-                          :hint "unable to send events"
-                          :context resp))))
+                  resp    (http-client params {:sync? true})]
+              (if (= (:status resp) 204)
+                true
+                (do
+                  (l/error :hint "unable to archive events"
+                           :resp-status (:status resp)
+                           :resp-body (:body resp))
+                  false))))
 
           (mark-as-archived [conn rows]
             (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)"
@@ -212,30 +332,16 @@
             xform  (comp (map decode-row)
                          (map row->event))
             events (into [] xform rows)]
-        (l/debug :action "archive-events" :uri uri :events (count events))
-        (if (empty? events)
-          :empty
-          (do
-            (send events)
+        (when-not (empty? events)
+          (l/debug :action "archive-events" :uri uri :events (count events))
+          (when (send events)
             (mark-as-archived conn rows)
-            :continue))))))
+            (count events)))))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; GC Task
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(declare clean-archived)
-
-(s/def ::max-age ::cf/audit-archive-gc-max-age)
-
-(defmethod ig/pre-init-spec ::archive-gc-task [_]
-  (s/keys :req-un [::db/pool ::enabled ::max-age]))
-
-(defmethod ig/init-key ::archive-gc-task
-  [_ cfg]
-  (fn [_]
-    (clean-archived cfg)))
-
 (def sql:clean-archived
   "delete from audit_log
     where archived_at is not null
@@ -248,3 +354,13 @@
         result   (:next.jdbc/update-count result)]
     (l/debug :action "clean archived audit log" :removed result)
     result))
+
+(s/def ::max-age ::cf/audit-log-gc-max-age)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool ::max-age]))
+
+(defmethod ig/init-key ::gc-task
+  [_ cfg]
+  (fn [_]
+    (clean-archived cfg)))

backend/src/app/loggers/database.clj

@@ -0,0 +1,92 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.database
+  "A specific logger impl that persists errors on the database."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare handle-event)
+
+(defonce enabled (atom true))
+
+(defn- persist-on-database!
+  [{:keys [pool] :as cfg} {:keys [id] :as event}]
+  (when-not (db/read-only? pool)
+    (db/insert! pool :server-error-report {:id id :content (db/tjson event)})))
+
+(defn- parse-event-data
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   event))
+
+(defn parse-event
+  [event]
+  (-> (parse-event-data event)
+      (assoc :tenant (cf/get :tenant))
+      (assoc :host (cf/get :host))
+      (assoc :public-uri (cf/get :public-uri))
+      (assoc :version (:full cf/version))
+      (update :id #(or % (uuid/next)))))
+
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [event (parse-event event)
+            uri   (cf/get :public-uri)]
+
+        (l/debug :hint "registering error on database" :id (:id event)
+                 :uri (str uri "/dbg/error/" (:id event)))
+
+        (persist-on-database! cfg event))
+      (catch Exception cause
+        (l/warn :hint "unexpected exception on database error logger" :cause cause)))))
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
+
+(defn error-event?
+  [event]
+  (= "error" (:logger/level event)))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver] :as cfg}]
+  (l/info :msg "initializing database error persistence")
+  (let [output (a/chan (a/sliding-buffer 5) (filter error-event?))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [msg (a/<! output)]
+        (if (nil? msg)
+          (l/info :msg "stoping error reporting loop")
+          (do
+            (a/<! (handle-event cfg msg))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (a/close! output))

backend/src/app/loggers/loki.clj

@@ -7,39 +7,37 @@
 (ns app.loggers.loki
   "A Loki integration."
   (:require
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.config :as cfg]
-   [app.util.async :as aa]
-   [app.util.http :as http]
    [app.util.json :as json]
-   [app.util.logging :as l]
-   [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
 
-(declare handle-event)
+(declare ^:private handle-event)
+(declare ^:private start-rcv-loop)
 
 (s/def ::uri ::us/string)
 (s/def ::receiver fn?)
+(s/def ::http-client fn?)
 
 (defmethod ig/pre-init-spec ::reporter [_]
-  (s/keys :req-un [::wrk/executor ::receiver]
+  (s/keys :req-un [ ::receiver ::http-client]
           :opt-un [::uri]))
 
 (defmethod ig/init-key ::reporter
   [_ {:keys [receiver uri] :as cfg}]
   (when uri
-    (l/info :msg "intializing loki reporter" :uri uri)
-    (let [input (a/chan (a/dropping-buffer 512))]
+    (l/info :msg "initializing loki reporter" :uri uri)
+    (let [input (a/chan (a/dropping-buffer 2048))]
       (receiver :sub input)
-      (a/go-loop []
-        (let [msg (a/<! input)]
-          (if (nil? msg)
-            (l/info :msg "stoping error reporting loop")
-            (do
-              (a/<! (handle-event cfg msg))
-              (recur)))))
+
+      (doto (Thread. #(start-rcv-loop cfg input))
+        (.setDaemon true)
+        (.setName "penpot/loki-sender")
+        (.start))
+
       input)))
 
 (defmethod ig/halt-key! ::reporter
@@ -47,53 +45,49 @@
   (when output
     (a/close! output)))
 
+(defn- start-rcv-loop
+  [cfg input]
+  (loop []
+    (let [msg (a/<!! input)]
+      (when-not (nil? msg)
+        (handle-event cfg msg)
+        (recur))))
+
+  (l/info :msg "stoping error reporting loop"))
+
 (defn- prepare-payload
   [event]
   (let [labels {:host    (cfg/get :host)
                 :tenant  (cfg/get :tenant)
                 :version (:full cfg/version)
-                :logger  (:logger event)
-                :level   (:level event)}]
+                :logger  (:logger/name event)
+                :level   (:logger/level event)}]
     {:streams
      [{:stream labels
        :values [[(str (* (inst-ms (:created-at event)) 1000000))
                  (str (:message event)
-                      (when-let [error (:error event)]
-                        (str "\n" (:trace error))))]]}]}))
+                      (when-let [error (:trace event)]
+                        (str "\n" error)))]]}]}))
 
-(defn- send-log
-  [uri payload i]
-  (try
-    (let [response (http/send! {:uri uri
-                                :timeout 6000
-                                :method :post
-                                :headers {"content-type" "application/json"}
-                                :body (json/encode payload)})]
-      (cond
-        (= (:status response) 204)
-        true
 
-        (= (:status response) 400)
-        (do
-          (l/error :hint "error on sending log to loki (no retry)"
-                   :rsp (pr-str response))
-          true)
-
-        :else
-        (do
-          (l/error :hint "error on sending log to loki" :try i
-                   :rsp (pr-str response))
-          false)))
-    (catch Exception e
-      (l/error :hint "error on sending message to loki" :cause e :try i)
-      false)))
+(defn- make-request
+  [{:keys [http-client uri] :as cfg} payload]
+  (http-client {:uri uri
+                :timeout 3000
+                :method :post
+                :headers {"content-type" "application/json"}
+                :body (json/write payload)}
+               {:sync? true}))
 
 (defn- handle-event
-  [{:keys [executor uri]} event]
-  (aa/with-thread executor
-    (let [payload (prepare-payload event)]
-      (loop [i 1]
-        (when (and (not (send-log uri payload i)) (< i 20))
-          (Thread/sleep (* i 2000))
-          (recur (inc i)))))))
-
+  [cfg event]
+  (try
+    (let [payload  (prepare-payload event)
+          response (make-request cfg payload)]
+      (when-not (= 204 (:status response))
+        (map? response)
+        (l/error :hint "error on sending log to loki (unexpected response)"
+                 :response (pr-str response))))
+    (catch Throwable cause
+      (l/error :hint "error on sending log to loki (unexpected exception)"
+               :cause cause))))

backend/src/app/loggers/mattermost.clj

@@ -7,35 +7,49 @@
 (ns app.loggers.mattermost
   "A mattermost integration for error reporting."
   (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.common.uuid :as uuid]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.util.async :as aa]
-   [app.util.http :as http]
+   [app.common.logging :as l]
+   [app.config :as cf]
+   [app.loggers.database :as ldb]
    [app.util.json :as json]
-   [app.util.logging :as l]
-   [app.util.template :as tmpl]
-   [app.worker :as wrk]
    [clojure.core.async :as a]
-   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Error Listener
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defonce enabled (atom true))
 
-(declare handle-event)
+(defn- send-mattermost-notification!
+  [{:keys [http-client] :as cfg} {:keys [host id public-uri] :as event}]
+  (let [uri  (:uri cfg)
+        text (str "Exception on (host: " host ", url: " public-uri "/dbg/error/" id ")\n"
+                  (when-let [pid (:profile-id event)]
+                    (str "- profile-id: #uuid-" pid "\n")))]
+    (p/then
+     (http-client {:uri uri
+                   :method :post
+                   :headers {"content-type" "application/json"}
+                   :body (json/write-str {:text text})})
+     (fn [{:keys [status] :as rsp}]
+       (when (not= status 200)
+         (l/warn :hint "error on sending data to mattermost"
+                 :response (pr-str rsp)))))))
 
-(defonce enabled-mattermost (atom true))
+(defn handle-event
+  [cfg event]
+  (let [ch (a/chan)]
+    (-> (p/let [event (ldb/parse-event event)]
+          (send-mattermost-notification! cfg event))
+        (p/finally (fn [_ cause]
+                     (when cause
+                       (l/warn :hint "unexpected exception on error reporter" :cause cause))
+                     (a/close! ch))))
+    ch))
 
-(s/def ::uri ::us/string)
+(s/def ::http-client fn?)
+(s/def ::uri ::cf/error-report-webhook)
 
 (defmethod ig/pre-init-spec ::reporter [_]
-  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+  (s/keys :req-un [::http-client ::receiver]
           :opt-un [::uri]))
 
 (defmethod ig/init-key ::reporter
@@ -43,7 +57,8 @@
   (when uri
     (l/info :msg "initializing mattermost error reporter" :uri uri)
     (let [output (a/chan (a/sliding-buffer 128)
-                         (filter #(= (:level %) "error")))]
+                         (filter (fn [event]
+                                   (= (:logger/level event) "error"))))]
       (receiver :sub output)
       (a/go-loop []
         (let [msg (a/<! output)]
@@ -58,95 +73,3 @@
   [_ output]
   (when output
     (a/close! output)))
-
-(defn- send-mattermost-notification!
-  [cfg {:keys [host id] :as cdata}]
-  (try
-    (let [uri  (:uri cfg)
-          text (str "Unhandled exception (host: " host ", url: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
-                    "- profile-id: #" (:profile-id cdata) "\n")
-          rsp  (http/send! {:uri uri
-                            :method :post
-                            :headers {"content-type" "application/json"}
-                            :body (json/encode-str {:text text})})]
-      (when (not= (:status rsp) 200)
-        (l/error :hint "error on sending data to mattermost"
-                 :response (pr-str rsp))))
-
-    (catch Exception e
-      (l/error :hint "unexpected exception on error reporter"
-               :cause e))))
-
-(defn- persist-on-database!
-  [{:keys [pool] :as cfg} {:keys [id] :as cdata}]
-  (db/with-atomic [conn pool]
-    (db/insert! conn :server-error-report
-                {:id id :content (db/tjson cdata)})))
-
-(defn- parse-context
-  [event]
-  (reduce-kv
-   (fn [acc k v]
-     (cond
-       (= k :id)         (assoc acc k (uuid/uuid v))
-       (= k :profile-id) (assoc acc k (uuid/uuid v))
-       (str/blank? v)    acc
-       :else             (assoc acc k v)))
-   {:id (uuid/next)}
-   (:context event)))
-
-(defn- parse-event
-  [event]
-  (-> (parse-context event)
-      (merge (dissoc event :context))
-      (assoc :tenant (cfg/get :tenant))
-      (assoc :host (cfg/get :host))
-      (assoc :public-uri (cfg/get :public-uri))
-      (assoc :version (:full cfg/version))))
-
-(defn handle-event
-  [{:keys [executor] :as cfg} event]
-  (aa/with-thread executor
-    (try
-      (let [cdata (parse-event event)]
-        (when @enabled-mattermost
-          (send-mattermost-notification! cfg cdata))
-        (persist-on-database! cfg cdata))
-      (catch Exception e
-        (l/error :hint "unexpected exception on error reporter"
-                 :cause e)))))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Handler
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::db/pool]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [pool] :as cfg}]
-  (letfn [(parse-id [request]
-            (let [id (get-in request [:path-params :id])
-                  id (us/uuid-conformer id)]
-              (when (uuid? id)
-                id)))
-          (retrieve-report [id]
-            (ex/ignoring
-             (when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
-               (assoc row :content (db/decode-transit-pgobject content)))))
-
-          (render-template [{:keys [content] :as report}]
-            (some-> (io/resource "error-report.tmpl")
-                    (tmpl/render content)))]
-
-
-    (fn [request]
-      (let [result (some-> (parse-id request)
-                           (retrieve-report)
-                           (render-template))]
-        (if result
-          {:status 200
-           :headers {"content-type" "text/html; charset=utf-8"}
-           :body result}
-          {:status 404
-           :body "not found"})))))

backend/src/app/loggers/sentry.clj

@@ -0,0 +1,170 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.sentry
+  "A mattermost integration for error reporting."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig])
+  (:import
+   io.sentry.Scope
+   io.sentry.IHub
+   io.sentry.Hub
+   io.sentry.NoOpHub
+   io.sentry.protocol.User
+   io.sentry.SentryOptions
+   io.sentry.SentryLevel
+   io.sentry.ScopeCallback))
+
+(defonce enabled (atom true))
+
+(defn- parse-context
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {}
+   (:context event)))
+
+(defn- parse-event
+  [event]
+  (assoc event :context (parse-context event)))
+
+(defn- build-sentry-options
+  [cfg]
+  (let [version (:base cf/version)]
+    (doto (SentryOptions.)
+      (.setDebug (:debug cfg false))
+      (.setTracesSampleRate (:traces-sample-rate cfg 1.0))
+      (.setDsn (:dsn cfg))
+      (.setServerName (cf/get :host))
+      (.setEnvironment (cf/get :tenant))
+      (.setAttachServerName true)
+      (.setAttachStacktrace (:attach-stack-trace cfg false))
+      (.setRelease (str "backend@" (if (= version "0.0.0") "develop" version))))))
+
+(defn handle-event
+  [^IHub shub event]
+  (letfn [(set-user! [^Scope scope {:keys [context] :as event}]
+            (let [user (User.)]
+              (.setIpAddress ^User user ^String (:ip-addr context))
+              (when-let [pid (:profile-id context)]
+                (.setId ^User user ^String (str pid)))
+              (.setUser scope ^User user)))
+
+          (set-level! [^Scope scope]
+            (.setLevel scope SentryLevel/ERROR))
+
+          (set-context! [^Scope scope {:keys [context] :as event}]
+            (let [uri (str (cf/get :public-uri) "/dbg/error-by-id/" (:id context))]
+              (.setContexts scope "detailed_error_uri" ^String uri))
+            (when-let [vers (:frontend-version event)]
+              (.setContexts scope "frontend_version" ^String vers))
+            (when-let [puri (:public-uri event)]
+              (.setContexts scope "public_uri" ^String (str puri)))
+            (when-let [uagent (:user-agent context)]
+              (.setContexts scope "user_agent" ^String uagent))
+            (when-let [tenant (:tenant event)]
+              (.setTag scope "tenant" ^String tenant))
+            (when-let [type (:error-type context)]
+              (.setTag scope "error_type" ^String  (str type)))
+            (when-let [code (:error-code context)]
+              (.setTag scope "error_code" ^String (str code)))
+            )
+
+          (capture [^Scope scope {:keys [context error] :as event}]
+            (let [msg   (str (:message error) "\n\n"
+
+                             "======================================================\n"
+                             "=================== Params ===========================\n"
+                             "======================================================\n"
+
+                             (:params context) "\n"
+
+                             (when (:explain context)
+                               (str "======================================================\n"
+                                    "=================== Explain ==========================\n"
+                                    "======================================================\n"
+                                    (:explain context) "\n"))
+
+                             (when (:data context)
+                               (str "======================================================\n"
+                                    "=================== Error Data =======================\n"
+                                    "======================================================\n"
+                                    (:data context) "\n"))
+
+                             (str "======================================================\n"
+                                  "=================== Stack Trace ======================\n"
+                                  "======================================================\n"
+                                  (:trace error))
+
+                             "\n")]
+              (set-user! scope event)
+              (set-level! scope)
+              (set-context! scope event)
+              (.captureMessage ^IHub shub msg)
+              ))
+          ]
+    (when @enabled
+      (.withScope ^IHub shub (reify ScopeCallback
+                               (run [_ scope]
+                                 (->> event
+                                      (parse-event)
+                                      (capture scope))))))
+
+    ))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::receiver any?)
+(s/def ::dsn ::cf/sentry-dsn)
+(s/def ::trace-sample-rate ::cf/sentry-trace-sample-rate)
+(s/def ::attach-stack-trace ::cf/sentry-attach-stack-trace)
+(s/def ::debug ::cf/sentry-debug)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+          :opt-un [::dsn ::trace-sample-rate ::attach-stack-trace]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver dsn executor] :as cfg}]
+  (l/info :msg "initializing sentry reporter" :dsn dsn)
+  (let [opts   (build-sentry-options cfg)
+        shub   (if dsn
+                 (Hub. ^SentryOptions opts)
+                 (NoOpHub/getInstance))
+        output (a/chan (a/sliding-buffer 128)
+                       (filter #(= (:level %) "error")))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [event (a/<! output)]
+        (if (nil? event)
+          (do
+            (l/info :msg "stoping error reporting loop")
+            (.close ^IHub shub))
+          (do
+            (a/<! (aa/with-thread executor (handle-event shub event)))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))

backend/src/app/loggers/zmq.clj

@@ -7,10 +7,10 @@
 (ns app.loggers.zmq
   "A generic ZMQ listener."
   (:require
-   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.util.json :as json]
-   [app.util.logging :as l]
    [app.util.time :as dt]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
@@ -31,13 +31,17 @@
 
 (defmethod ig/init-key ::receiver
   [_ {:keys [endpoint] :as cfg}]
-  (l/info :msg "intializing ZMQ receiver" :bind endpoint)
+  (l/info :msg "initializing ZMQ receiver" :bind endpoint)
   (let [buffer (a/chan 1)
         output (a/chan 1 (comp (filter map?)
-                               (map prepare)))
+                               (keep prepare)))
         mult   (a/mult output)]
     (when endpoint
-      (a/thread (start-rcv-loop {:out buffer :endpoint endpoint})))
+      (let [thread (Thread. #(start-rcv-loop {:out buffer :endpoint endpoint}))]
+        (.setDaemon thread false)
+        (.setName thread "penpot/zmq-logger-receiver")
+        (.start thread)))
+
     (a/pipe buffer output)
     (with-meta
       (fn [cmd ch]
@@ -53,37 +57,54 @@
   [_ f]
   (a/close! (::buffer (meta f))))
 
+(def ^:private json-mapper
+  (json/mapper
+   {:encode-key-fn str/camel
+    :decode-key-fn (comp keyword str/kebab)}))
+
 (defn- start-rcv-loop
   ([] (start-rcv-loop nil))
   ([{:keys [out endpoint] :or {endpoint "tcp://localhost:5556"}}]
    (let [out    (or out (a/chan 1))
-         zctx   (ZContext.)
+         zctx   (ZContext. 1)
          socket (.. zctx (createSocket SocketType/SUB))]
      (.. socket (connect ^String endpoint))
      (.. socket (subscribe ""))
      (.. socket (setReceiveTimeOut 5000))
      (loop []
        (let [msg (.recv ^ZMQ$Socket socket)
-             msg (json/decode msg)
+             msg (ex/ignoring (json/read msg json-mapper))
              msg (if (nil? msg) :empty msg)]
          (if (a/>!! out msg)
            (recur)
            (do
              (.close ^java.lang.AutoCloseable socket)
-             (.close ^java.lang.AutoCloseable zctx))))))))
+             (.destroy ^ZContext zctx))))))))
+
+(s/def ::logger-name string?)
+(s/def ::level string?)
+(s/def ::thread string?)
+(s/def ::time-millis integer?)
+(s/def ::message string?)
+(s/def ::context-map map?)
+(s/def ::thrown map?)
+
+(s/def ::log4j-event
+  (s/keys :req-un [::logger-name ::level ::thread ::time-millis ::message]
+          :opt-un [::context-map ::thrown]))
 
 (defn- prepare
   [event]
-  (d/merge
-   {:logger     (:loggerName event)
-    :level      (str/lower (:level event))
-    :thread     (:thread event)
-    :created-at (dt/instant (:timeMillis event))
-    :message    (:message event)}
-   (when-let [ctx (:contextMap event)]
-     {:context ctx})
-   (when-let [thrown (:thrown event)]
-     {:error
-      {:class (:name thrown)
-       :message (:message thrown)
-       :trace (:extendedStackTrace thrown)}})))
+  (if (s/valid? ::log4j-event event)
+    (merge {:message      (:message event)
+            :created-at   (dt/instant (:time-millis event))
+            :logger/name  (:logger-name event)
+            :logger/level (str/lower (:level event))}
+
+           (when-let [trace (-> event :thrown :extended-stack-trace)]
+             {:trace trace})
+
+           (:context-map event))
+    (do
+      (l/warn :hint "invalid event" :event event)
+      nil)))

backend/src/app/main.clj

@@ -6,233 +6,254 @@
 
 (ns app.main
   (:require
+   [app.auth.oidc]
+   [app.common.logging :as l]
    [app.config :as cf]
-   [app.util.logging :as l]
    [app.util.time :as dt]
-   [integrant.core :as ig]))
+   [integrant.core :as ig])
+  (:gen-class))
 
 (def system-config
   {:app.db/pool
    {:uri        (cf/get :database-uri)
     :username   (cf/get :database-username)
     :password   (cf/get :database-password)
+    :read-only  (cf/get :database-readonly false)
     :metrics    (ig/ref :app.metrics/metrics)
     :migrations (ig/ref :app.migrations/all)
-    :name :main
-    :min-pool-size 0
-    :max-pool-size 20}
+    :name       :main
+    :min-size   (cf/get :database-min-pool-size 0)
+    :max-size   (cf/get :database-max-pool-size 30)}
 
-   :app.metrics/metrics
-   {:definitions
-    {:profile-register
-     {:name "actions_profile_register_count"
-      :help "A global counter of user registrations."
-      :type :counter}
-     :profile-activation
-     {:name "actions_profile_activation_count"
-      :help "A global counter of profile activations"
-      :type :counter}}}
+   ;; Default thread pool for IO operations
+   [::default :app.worker/executor]
+   {:parallelism (cf/get :default-executor-parallelism 60)
+    :prefix :default}
 
-   :app.migrations/all
-   {:main (ig/ref :app.migrations/migrations)}
+   ;; Constrained thread pool. Should only be used from high resources
+   ;; demanding operations.
+   [::blocking :app.worker/executor]
+   {:parallelism (cf/get :blocking-executor-parallelism 10)
+    :prefix :blocking}
+
+   ;; Dedicated thread pool for backround tasks execution.
+   [::worker :app.worker/executor]
+   {:parallelism (cf/get :worker-executor-parallelism 10)
+    :prefix :worker}
+
+   :app.worker/scheduler
+   {:parallelism 1
+    :prefix :scheduler}
+
+   :app.worker/executors
+   {:default  (ig/ref [::default :app.worker/executor])
+    :worker   (ig/ref [::worker :app.worker/executor])
+    :blocking (ig/ref [::blocking :app.worker/executor])}
+
+   :app.worker/executors-monitor
+   {:metrics   (ig/ref :app.metrics/metrics)
+    :scheduler (ig/ref :app.worker/scheduler)
+    :executors (ig/ref :app.worker/executors)}
 
    :app.migrations/migrations
    {}
 
+   :app.metrics/metrics
+   {}
+
+   :app.migrations/all
+   {:main (ig/ref :app.migrations/migrations)}
+
    :app.msgbus/msgbus
    {:backend   (cf/get :msgbus-backend :redis)
+    :executor  (ig/ref [::default :app.worker/executor])
     :redis-uri (cf/get :redis-uri)}
 
    :app.tokens/tokens
    {:keys (ig/ref :app.setup/keys)}
 
+   :app.storage.tmp/cleaner
+   {:executor (ig/ref [::worker :app.worker/executor])
+    :scheduler (ig/ref :app.worker/scheduler)}
+
    :app.storage/gc-deleted-task
    {:pool     (ig/ref :app.db/pool)
     :storage  (ig/ref :app.storage/storage)
-    :min-age  (dt/duration {:hours 2})}
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.storage/gc-touched-task
-   {:pool     (ig/ref :app.db/pool)}
+   {:pool (ig/ref :app.db/pool)}
 
-   :app.storage/recheck-task
+   :app.http/client
+   {:executor (ig/ref [::default :app.worker/executor])}
+
+   :app.http/session
+   {:store (ig/ref :app.http.session/store)}
+
+   :app.http.session/store
    {:pool     (ig/ref :app.db/pool)
-    :storage  (ig/ref :app.storage/storage)}
-
-   :app.http.session/session
-   {:pool   (ig/ref :app.db/pool)
-    :tokens (ig/ref :app.tokens/tokens)}
+    :tokens   (ig/ref :app.tokens/tokens)
+    :executor (ig/ref [::default :app.worker/executor])}
 
    :app.http.session/gc-task
    {:pool        (ig/ref :app.db/pool)
-    :max-age     (cf/get :http-session-idle-max-age)}
-
-   :app.http.session/updater
-   {:pool           (ig/ref :app.db/pool)
-    :metrics        (ig/ref :app.metrics/metrics)
-    :executor       (ig/ref :app.worker/executor)
-    :session        (ig/ref :app.http.session/session)
-    :max-batch-age  (cf/get :http-session-updater-batch-max-age)
-    :max-batch-size (cf/get :http-session-updater-batch-max-size)}
+    :max-age     (cf/get :auth-token-cookie-max-age)}
 
    :app.http.awsns/handler
-   {:tokens  (ig/ref :app.tokens/tokens)
-    :pool    (ig/ref :app.db/pool)}
+   {:tokens      (ig/ref :app.tokens/tokens)
+    :pool        (ig/ref :app.db/pool)
+    :http-client (ig/ref :app.http/client)
+    :executor    (ig/ref [::worker :app.worker/executor])}
 
    :app.http/server
-   {:port    (cf/get :http-server-port)
-    :router  (ig/ref :app.http/router)
-    :metrics (ig/ref :app.metrics/metrics)
-    :ws      {"/ws/notifications" (ig/ref :app.notifications/handler)}}
+   {:port        (cf/get :http-server-port)
+    :host        (cf/get :http-server-host)
+    :router      (ig/ref :app.http/router)
+    :metrics     (ig/ref :app.metrics/metrics)
+    :executor    (ig/ref [::default :app.worker/executor])
+    :io-threads  (cf/get :http-server-io-threads)
+    :max-body-size           (cf/get :http-server-max-body-size)
+    :max-multipart-body-size (cf/get :http-server-max-multipart-body-size)}
+
+   :app.auth.ldap/provider
+   {:host           (cf/get :ldap-host)
+    :port           (cf/get :ldap-port)
+    :ssl            (cf/get :ldap-ssl)
+    :tls            (cf/get :ldap-starttls)
+    :query          (cf/get :ldap-user-query)
+    :attrs-email    (cf/get :ldap-attrs-email)
+    :attrs-fullname (cf/get :ldap-attrs-fullname)
+    :attrs-username (cf/get :ldap-attrs-username)
+    :base-dn        (cf/get :ldap-base-dn)
+    :bind-dn        (cf/get :ldap-bind-dn)
+    :bind-password  (cf/get :ldap-bind-password)
+    :enabled?       (contains? cf/flags :login-with-ldap)}
+
+   :app.auth.oidc/google-provider
+   {:enabled?      (contains? cf/flags :login-with-google)
+    :client-id     (cf/get :google-client-id)
+    :client-secret (cf/get :google-client-secret)}
+
+   :app.auth.oidc/github-provider
+   {:enabled?      (contains? cf/flags :login-with-github)
+    :http-client   (ig/ref :app.http/client)
+    :client-id     (cf/get :github-client-id)
+    :client-secret (cf/get :github-client-secret)}
+
+   :app.auth.oidc/gitlab-provider
+   {:enabled?      (contains? cf/flags :login-with-gitlab)
+    :base-uri      (cf/get :gitlab-base-uri "https://gitlab.com")
+    :client-id     (cf/get :gitlab-client-id)
+    :client-secret (cf/get :gitlab-client-secret)}
+
+   :app.auth.oidc/generic-provider
+   {:enabled?      (contains? cf/flags :login-with-oidc)
+    :http-client   (ig/ref :app.http/client)
+
+    :client-id     (cf/get :oidc-client-id)
+    :client-secret (cf/get :oidc-client-secret)
+
+    :base-uri      (cf/get :oidc-base-uri)
+
+    :token-uri     (cf/get :oidc-token-uri)
+    :auth-uri      (cf/get :oidc-auth-uri)
+    :user-uri      (cf/get :oidc-user-uri)
+
+    :scopes        (cf/get :oidc-scopes)
+    :roles-attr    (cf/get :oidc-roles-attr)
+    :roles         (cf/get :oidc-roles)}
+
+   :app.auth.oidc/routes
+   {:providers   {:google (ig/ref :app.auth.oidc/google-provider)
+                  :github (ig/ref :app.auth.oidc/github-provider)
+                  :gitlab (ig/ref :app.auth.oidc/gitlab-provider)
+                  :oidc   (ig/ref :app.auth.oidc/generic-provider)}
+    :tokens      (ig/ref :app.tokens/tokens)
+    :http-client (ig/ref :app.http/client)
+    :pool        (ig/ref :app.db/pool)
+    :session     (ig/ref :app.http/session)
+    :public-uri  (cf/get :public-uri)
+    :executor    (ig/ref [::default :app.worker/executor])}
 
    :app.http/router
-   {:rpc         (ig/ref :app.rpc/rpc)
-    :session     (ig/ref :app.http.session/session)
-    :tokens      (ig/ref :app.tokens/tokens)
-    :public-uri  (cf/get :public-uri)
-    :metrics     (ig/ref :app.metrics/metrics)
-    :oauth       (ig/ref :app.http.oauth/handler)
-    :assets      (ig/ref :app.http.assets/handlers)
-    :storage     (ig/ref :app.storage/storage)
-    :sns-webhook (ig/ref :app.http.awsns/handler)
-    :feedback    (ig/ref :app.http.feedback/handler)
-    :error-report-handler (ig/ref :app.loggers.mattermost/handler)}
+   {:assets        (ig/ref :app.http.assets/handlers)
+    :feedback      (ig/ref :app.http.feedback/handler)
+    :session       (ig/ref :app.http/session)
+    :awsns-handler (ig/ref :app.http.awsns/handler)
+    :debug-routes  (ig/ref :app.http.debug/routes)
+    :oidc-routes   (ig/ref :app.auth.oidc/routes)
+    :ws            (ig/ref :app.http.websocket/handler)
+    :metrics       (ig/ref :app.metrics/metrics)
+    :public-uri    (cf/get :public-uri)
+    :storage       (ig/ref :app.storage/storage)
+    :tokens        (ig/ref :app.tokens/tokens)
+    :audit-handler (ig/ref :app.loggers.audit/http-handler)
+    :rpc-routes    (ig/ref :app.rpc/routes)
+    :doc-routes    (ig/ref :app.rpc.doc/routes)
+    :executor      (ig/ref [::default :app.worker/executor])}
+
+   :app.http.debug/routes
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])
+    :storage  (ig/ref :app.storage/storage)
+    :session  (ig/ref :app.http/session)}
+
+   :app.http.websocket/handler
+   {:pool     (ig/ref :app.db/pool)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :msgbus   (ig/ref :app.msgbus/msgbus)}
 
    :app.http.assets/handlers
    {:metrics           (ig/ref :app.metrics/metrics)
     :assets-path       (cf/get :assets-path)
     :storage           (ig/ref :app.storage/storage)
+    :executor          (ig/ref [::default :app.worker/executor])
     :cache-max-age     (dt/duration {:hours 24})
     :signature-max-age (dt/duration {:hours 24 :minutes 5})}
 
    :app.http.feedback/handler
-   {:pool (ig/ref :app.db/pool)}
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::default :app.worker/executor])}
 
-   :app.http.oauth/handler
-   {:rpc           (ig/ref :app.rpc/rpc)
-    :session       (ig/ref :app.http.session/session)
-    :pool          (ig/ref :app.db/pool)
-    :tokens        (ig/ref :app.tokens/tokens)
-    :audit         (ig/ref :app.loggers.audit/collector)
-    :public-uri    (cf/get :public-uri)}
+   :app.rpc/methods
+   {:pool        (ig/ref :app.db/pool)
+    :session     (ig/ref :app.http/session)
+    :tokens      (ig/ref :app.tokens/tokens)
+    :metrics     (ig/ref :app.metrics/metrics)
+    :storage     (ig/ref :app.storage/storage)
+    :msgbus      (ig/ref :app.msgbus/msgbus)
+    :public-uri  (cf/get :public-uri)
+    :audit       (ig/ref :app.loggers.audit/collector)
+    :ldap        (ig/ref :app.auth.ldap/provider)
+    :http-client (ig/ref :app.http/client)
+    :executors   (ig/ref :app.worker/executors)}
 
-   ;; RLimit definition for password hashing
-   :app.rlimits/password
-   (cf/get :rlimits-password)
+   :app.rpc.doc/routes
+   {:methods (ig/ref :app.rpc/methods)}
 
-   ;; RLimit definition for image processing
-   :app.rlimits/image
-   (cf/get :rlimits-image)
-
-   ;; RLimit definition for font processing
-   :app.rlimits/font
-   (cf/get :rlimits-font 2)
-
-   ;; A collection of rlimits as hash-map.
-   :app.rlimits/all
-   {:password (ig/ref :app.rlimits/password)
-    :image    (ig/ref :app.rlimits/image)
-    :font     (ig/ref :app.rlimits/font)}
-
-   :app.rpc/rpc
-   {:pool       (ig/ref :app.db/pool)
-    :session    (ig/ref :app.http.session/session)
-    :tokens     (ig/ref :app.tokens/tokens)
-    :metrics    (ig/ref :app.metrics/metrics)
-    :storage    (ig/ref :app.storage/storage)
-    :msgbus     (ig/ref :app.msgbus/msgbus)
-    :rlimits    (ig/ref :app.rlimits/all)
-    :public-uri (cf/get :public-uri)
-    :audit      (ig/ref :app.loggers.audit/collector)}
-
-   :app.notifications/handler
-   {:msgbus   (ig/ref :app.msgbus/msgbus)
-    :pool     (ig/ref :app.db/pool)
-    :session  (ig/ref :app.http.session/session)
-    :metrics  (ig/ref :app.metrics/metrics)
-    :executor (ig/ref :app.worker/executor)}
-
-   :app.worker/executor
-   {:min-threads 0
-    :max-threads 256
-    :idle-timeout 60000
-    :name :worker}
-
-   :app.worker/worker
-   {:executor (ig/ref :app.worker/executor)
-    :tasks    (ig/ref :app.worker/registry)
-    :metrics  (ig/ref :app.metrics/metrics)
-    :pool     (ig/ref :app.db/pool)}
-
-   :app.worker/scheduler
-   {:executor   (ig/ref :app.worker/executor)
-    :tasks      (ig/ref :app.worker/registry)
-    :pool       (ig/ref :app.db/pool)
-    :schedule
-    [{:cron #app/cron "0 0 0 * * ?" ;; daily
-      :task :file-media-gc}
-
-     {:cron #app/cron "0 0 * * * ?"  ;; hourly
-      :task :file-xlog-gc}
-
-     {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :storage-deleted-gc}
-
-     {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :storage-touched-gc}
-
-     {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :session-gc}
-
-     {:cron #app/cron "0 0 * * * ?"  ;; hourly
-      :task :storage-recheck}
-
-     {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :objects-gc}
-
-     {:cron #app/cron "0 0 0 * * ?"  ;; daily
-      :task :tasks-gc}
-
-     (when (cf/get :fdata-storage-backed)
-       {:cron #app/cron "0 0 * * * ?"  ;; hourly
-        :task :file-offload})
-
-     (when (cf/get :audit-archive-enabled)
-       {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive})
-
-     (when (cf/get :audit-archive-gc-enabled)
-       {:cron #app/cron "0 0 * * * ?" ;; every 1h
-        :task :audit-archive-gc})
-
-     (when (cf/get :telemetry-enabled)
-       {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
-        :task :telemetry})]}
+   :app.rpc/routes
+   {:methods (ig/ref :app.rpc/methods)}
 
    :app.worker/registry
    {:metrics (ig/ref :app.metrics/metrics)
     :tasks
     {:sendmail           (ig/ref :app.emails/sendmail-handler)
      :objects-gc         (ig/ref :app.tasks.objects-gc/handler)
-     :delete-object      (ig/ref :app.tasks.delete-object/handler)
-     :delete-profile     (ig/ref :app.tasks.delete-profile/handler)
-     :file-media-gc      (ig/ref :app.tasks.file-media-gc/handler)
+     :file-gc            (ig/ref :app.tasks.file-gc/handler)
      :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
-     :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
-     :storage-touched-gc (ig/ref :app.storage/gc-touched-task)
-     :storage-recheck    (ig/ref :app.storage/recheck-task)
+     :storage-gc-deleted (ig/ref :app.storage/gc-deleted-task)
+     :storage-gc-touched (ig/ref :app.storage/gc-touched-task)
      :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
      :telemetry          (ig/ref :app.tasks.telemetry/handler)
      :session-gc         (ig/ref :app.http.session/gc-task)
-     :file-offload       (ig/ref :app.tasks.file-offload/handler)
-     :audit-archive      (ig/ref :app.loggers.audit/archive-task)
-     :audit-archive-gc   (ig/ref :app.loggers.audit/archive-gc-task)}}
+     :audit-log-archive  (ig/ref :app.loggers.audit/archive-task)
+     :audit-log-gc       (ig/ref :app.loggers.audit/gc-task)}}
 
    :app.emails/sendmail-handler
    {:host             (cf/get :smtp-host)
     :port             (cf/get :smtp-port)
     :ssl              (cf/get :smtp-ssl)
     :tls              (cf/get :smtp-tls)
-    :enabled          (cf/get :smtp-enabled)
     :username         (cf/get :smtp-username)
     :password         (cf/get :smtp-password)
     :metrics          (ig/ref :app.metrics/metrics)
@@ -243,37 +264,22 @@
    {:pool    (ig/ref :app.db/pool)
     :max-age cf/deletion-delay}
 
-   :app.tasks.delete-object/handler
+   :app.tasks.objects-gc/handler
    {:pool    (ig/ref :app.db/pool)
     :storage (ig/ref :app.storage/storage)}
 
-   :app.tasks.objects-gc/handler
-   {:pool    (ig/ref :app.db/pool)
-    :storage (ig/ref :app.storage/storage)
-    :max-age cf/deletion-delay}
-
-   :app.tasks.delete-profile/handler
-   {:pool    (ig/ref :app.db/pool)}
-
-   :app.tasks.file-media-gc/handler
-   {:pool    (ig/ref :app.db/pool)
-    :max-age cf/deletion-delay}
+   :app.tasks.file-gc/handler
+   {:pool (ig/ref :app.db/pool)}
 
    :app.tasks.file-xlog-gc/handler
-   {:pool    (ig/ref :app.db/pool)
-    :max-age (dt/duration {:hours 72})}
-
-   :app.tasks.file-offload/handler
-   {:pool    (ig/ref :app.db/pool)
-    :max-age (dt/duration {:seconds 5})
-    :storage (ig/ref :app.storage/storage)
-    :backend (cf/get :fdata-storage-backed :fdata-s3)}
+   {:pool (ig/ref :app.db/pool)}
 
    :app.tasks.telemetry/handler
    {:pool        (ig/ref :app.db/pool)
     :version     (:full cf/version)
     :uri         (cf/get :telemetry-uri)
-    :sprops      (ig/ref :app.setup/props)}
+    :sprops      (ig/ref :app.setup/props)
+    :http-client (ig/ref :app.http/client)}
 
    :app.srepl/server
    {:port (cf/get :srepl-port)
@@ -289,77 +295,117 @@
    :app.loggers.zmq/receiver
    {:endpoint (cf/get :loggers-zmq-uri)}
 
+   :app.loggers.audit/http-handler
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::default :app.worker/executor])}
+
    :app.loggers.audit/collector
-   {:enabled  (cf/get :audit-enabled false)
-    :pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)}
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.loggers.audit/archive-task
-   {:uri      (cf/get :audit-archive-uri)
-    :enabled  (cf/get :audit-archive-enabled false)
-    :tokens   (ig/ref :app.tokens/tokens)
-    :pool     (ig/ref :app.db/pool)}
+   {:uri         (cf/get :audit-log-archive-uri)
+    :tokens      (ig/ref :app.tokens/tokens)
+    :pool        (ig/ref :app.db/pool)
+    :http-client (ig/ref :app.http/client)}
 
-   :app.loggers.audit/archive-gc-task
-   {:enabled  (cf/get :audit-archive-gc-enabled false)
-    :max-age  (cf/get :audit-archive-gc-max-age cf/deletion-delay)
+   :app.loggers.audit/gc-task
+   {:max-age  (cf/get :audit-log-gc-max-age cf/deletion-delay)
     :pool     (ig/ref :app.db/pool)}
 
    :app.loggers.loki/reporter
-   {:uri      (cf/get :loggers-loki-uri)
-    :receiver (ig/ref :app.loggers.zmq/receiver)
-    :executor (ig/ref :app.worker/executor)}
+   {:uri         (cf/get :loggers-loki-uri)
+    :receiver    (ig/ref :app.loggers.zmq/receiver)
+    :http-client (ig/ref :app.http/client)}
 
    :app.loggers.mattermost/reporter
-   {:uri      (cf/get :error-report-webhook)
-    :receiver (ig/ref :app.loggers.zmq/receiver)
-    :pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)}
+   {:uri         (cf/get :error-report-webhook)
+    :receiver    (ig/ref :app.loggers.zmq/receiver)
+    :http-client (ig/ref :app.http/client)}
 
-   :app.loggers.mattermost/handler
-   {:pool (ig/ref :app.db/pool)}
+   :app.loggers.database/reporter
+   {:receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref [::worker :app.worker/executor])}
 
    :app.storage/storage
    {:pool     (ig/ref :app.db/pool)
-    :executor (ig/ref :app.worker/executor)
+    :executor (ig/ref [::default :app.worker/executor])
 
-    :backends {
-               :assets-s3 (ig/ref [::assets :app.storage.s3/backend])
-               :assets-db (ig/ref [::assets :app.storage.db/backend])
-               :assets-fs (ig/ref [::assets :app.storage.fs/backend])
-               :tmp       (ig/ref [::tmp  :app.storage.fs/backend])
-               :fdata-s3  (ig/ref [::fdata :app.storage.s3/backend])
+    :backends
+    {:assets-s3 (ig/ref [::assets :app.storage.s3/backend])
+     :assets-fs (ig/ref [::assets :app.storage.fs/backend])
 
-               ;; keep this for backward compatibility
-               :s3        (ig/ref [::assets :app.storage.s3/backend])
-               :fs        (ig/ref [::assets :app.storage.fs/backend])}}
-
-   [::fdata :app.storage.s3/backend]
-   {:region (cf/get :storage-fdata-s3-region)
-    :bucket (cf/get :storage-fdata-s3-bucket)
-    :prefix (cf/get :storage-fdata-s3-prefix)}
+     ;; keep this for backward compatibility
+     :s3        (ig/ref [::assets :app.storage.s3/backend])
+     :fs        (ig/ref [::assets :app.storage.fs/backend])}}
 
    [::assets :app.storage.s3/backend]
-   {:region (cf/get :storage-assets-s3-region)
-    :bucket (cf/get :storage-assets-s3-bucket)}
+   {:region   (cf/get :storage-assets-s3-region)
+    :endpoint (cf/get :storage-assets-s3-endpoint)
+    :bucket   (cf/get :storage-assets-s3-bucket)
+    :executor (ig/ref [::default :app.worker/executor])}
 
    [::assets :app.storage.fs/backend]
    {:directory (cf/get :storage-assets-fs-directory)}
+   })
 
-   [::tmp :app.storage.fs/backend]
-   {:directory "/tmp/penpot"}
 
-   [::assets :app.storage.db/backend]
-   {:pool (ig/ref :app.db/pool)}})
+(def worker-config
+  {   :app.worker/cron
+   {:executor   (ig/ref [::worker :app.worker/executor])
+    :scheduler  (ig/ref :app.worker/scheduler)
+    :tasks      (ig/ref :app.worker/registry)
+    :pool       (ig/ref :app.db/pool)
+    :entries
+    [{:cron #app/cron "0 0 0 * * ?" ;; daily
+      :task :file-gc}
+
+     {:cron #app/cron "0 0 * * * ?"  ;; hourly
+      :task :file-xlog-gc}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :storage-gc-deleted}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :storage-gc-touched}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :session-gc}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :objects-gc}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :tasks-gc}
+
+     {:cron #app/cron "0 30 */3,23 * * ?"
+      :task :telemetry}
+
+     (when (contains? cf/flags :audit-log-archive)
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
+        :task :audit-log-archive})
+
+     (when (contains? cf/flags :audit-log-gc)
+       {:cron #app/cron "0 0 0 * * ?" ;; daily
+        :task :audit-log-gc})]}
+
+   :app.worker/worker
+   {:executor (ig/ref [::worker :app.worker/executor])
+    :tasks    (ig/ref :app.worker/registry)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :pool     (ig/ref :app.db/pool)}})
 
 (def system nil)
 
 (defn start
   []
-  (ig/load-namespaces system-config)
+  (ig/load-namespaces (merge system-config worker-config))
   (alter-var-root #'system (fn [sys]
                              (when sys (ig/halt! sys))
                              (-> system-config
+                                 (cond-> (contains? cf/flags :backend-worker)
+                                   (merge worker-config))
                                  (ig/prep)
                                  (ig/init))))
   (l/info :msg "welcome to penpot"

backend/src/app/media.clj

@@ -12,45 +12,44 @@
    [app.common.media :as cm]
    [app.common.spec :as us]
    [app.config :as cf]
-   [app.rlimits :as rlm]
-   [app.rpc.queries.svg :as svg]
+   [app.storage.tmp :as tmp]
+   [app.util.bytes :as bs]
+   [app.util.svg :as svg]
    [buddy.core.bytes :as bb]
    [buddy.core.codecs :as bc]
-   [clojure.java.io :as io]
    [clojure.java.shell :as sh]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [datoteka.core :as fs])
   (:import
-   java.io.ByteArrayInputStream
-   java.io.OutputStream
-   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(s/def ::image-content-type cm/valid-image-types)
-(s/def ::font-content-type cm/valid-font-types)
-
-(s/def :internal.http.upload/filename ::us/string)
-(s/def :internal.http.upload/size ::us/integer)
-(s/def :internal.http.upload/content-type ::us/string)
-(s/def :internal.http.upload/tempfile any?)
+(s/def ::path fs/path?)
+(s/def ::filename string?)
+(s/def ::size integer?)
+(s/def ::headers (s/map-of string? string?))
+(s/def ::mtype string?)
 
 (s/def ::upload
-  (s/keys :req-un [:internal.http.upload/filename
-                   :internal.http.upload/size
-                   :internal.http.upload/tempfile
-                   :internal.http.upload/content-type]))
+  (s/keys :req-un [::filename ::size ::path]
+          :opt-un [::mtype ::headers]))
 
-(defn validate-media-type
-  ([mtype] (validate-media-type mtype cm/valid-image-types))
-  ([mtype allowed]
-   (when-not (contains? allowed mtype)
+;; A subset of fields from the ::upload spec
+(s/def ::input
+  (s/keys :req-un [::path]
+          :opt-un [::mtype]))
+
+(defn validate-media-type!
+  ([upload] (validate-media-type! upload cm/valid-image-types))
+  ([upload allowed]
+   (when-not (contains? allowed (:mtype upload))
      (ex/raise :type :validation
                :code :media-type-not-allowed
-               :hint "Seems like you are uploading an invalid media object"))))
+               :hint "Seems like you are uploading an invalid media object"))
 
+   upload))
 
 (defmulti process :cmd)
 (defmulti process-error class)
@@ -66,39 +65,23 @@
   (throw error))
 
 (defn run
-  [{:keys [rlimits] :as cfg} {:keys [rlimit] :or {rlimit :image} :as params}]
-  (us/assert map? rlimits)
-  (let [rlimit (get rlimits rlimit)]
-    (when-not rlimit
-      (ex/raise :type :internal
-                :code :rlimit-not-configured
-                :hint ":image rlimit not configured"))
-    (try
-      (rlm/execute rlimit (process params))
-      (catch Throwable e
-        (process-error e)))))
+  [params]
+  (try
+    (process params)
+    (catch Throwable e
+      (process-error e))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; --- Thumbnails Generation
+;; IMAGE THUMBNAILS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(s/def ::cmd keyword?)
-
-(s/def ::path (s/or :path fs/path?
-                    :string string?
-                    :file fs/file?))
-
-(s/def ::input
-  (s/keys :req-un [::path]
-          :opt-un [::cm/mtype]))
-
 (s/def ::width integer?)
 (s/def ::height integer?)
 (s/def ::format #{:jpeg :webp :png})
 (s/def ::quality #(< 0 % 101))
 
 (s/def ::thumbnail-params
-  (s/keys :req-un [::cmd ::input ::format ::width ::height]))
+  (s/keys :req-un [::input ::format ::width ::height]))
 
 ;; Related info on how thumbnails generation
 ;;  http://www.imagemagick.org/Usage/thumbnails/
@@ -108,18 +91,16 @@
   (let [{:keys [path mtype]} input
         format (or (cm/mtype->format mtype) format)
         ext    (cm/format->extension format)
-        tmp    (fs/create-tempfile :suffix ext)]
+        tmp    (tmp/tempfile :prefix "penpot.media." :suffix ext)]
 
     (doto (ConvertCmd.)
       (.run operation (into-array (map str [path tmp]))))
 
-    (let [thumbnail-data (fs/slurp-bytes tmp)]
-      (fs/delete tmp)
-      (assoc params
-             :format format
-             :mtype  (cm/format->mtype format)
-             :size   (alength ^bytes thumbnail-data)
-             :data   (ByteArrayInputStream. thumbnail-data)))))
+    (assoc params
+           :format format
+           :mtype  (cm/format->mtype format)
+           :size   (fs/size tmp)
+           :data   tmp)))
 
 (defmethod process :generic-thumbnail
   [{:keys [quality width height] :as params}]
@@ -180,12 +161,12 @@
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      (let [info (some-> path slurp svg/parse get-basic-info-from-svg)]
+      (let [info (some-> path slurp svg/pre-process svg/parse get-basic-info-from-svg)]
         (when-not info
           (ex/raise :type :validation
                     :code :invalid-svg-file
                     :hint "uploaded svg does not provides dimensions"))
-        (assoc info :mtype mtype))
+        (merge input info))
 
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
@@ -198,9 +179,9 @@
         ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
         ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
         ;; any frame.
-        {:width  (.getPageWidth instance)
-         :height (.getPageHeight instance)
-         :mtype  mtype}))))
+        (assoc input
+               :width  (.getPageWidth instance)
+               :height (.getPageHeight instance))))))
 
 (defmethod process-error org.im4java.core.InfoException
   [error]
@@ -210,67 +191,60 @@
             :cause error))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; --- Fonts Generation
+;; FONTS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(def all-fotmats #{"font/woff2", "font/woff", "font/otf", "font/ttf"})
-
 (defmethod process :generate-fonts
   [{:keys [input] :as params}]
   (letfn [(ttf->otf [data]
-            (let [input-file  (fs/create-tempfile :prefix "penpot")
-                  output-file (fs/path (str input-file ".otf"))
-                  _           (with-open [out (io/output-stream input-file)]
-                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
-                                (.flush ^OutputStream out))
-                  res         (sh/sh "fontforge" "-lang=ff" "-c"
-                                     (str/fmt "Open('%s'); Generate('%s')"
-                                              (str input-file)
-                                              (str output-file)))]
+            (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
+                  foutput (fs/path (str finput ".otf"))
+                  _       (bs/write-to-file! data finput)
+                  res     (sh/sh "fontforge" "-lang=ff" "-c"
+                                 (str/fmt "Open('%s'); Generate('%s')"
+                                          (str finput)
+                                          (str foutput)))]
               (when (zero? (:exit res))
-                (fs/slurp-bytes output-file))))
-
+                foutput)))
 
           (otf->ttf [data]
-            (let [input-file  (fs/create-tempfile :prefix "penpot")
-                  output-file (fs/path (str input-file ".ttf"))
-                  _           (with-open [out (io/output-stream input-file)]
-                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
-                                (.flush ^OutputStream out))
-                  res         (sh/sh "fontforge" "-lang=ff" "-c"
-                                     (str/fmt "Open('%s'); Generate('%s')"
-                                              (str input-file)
-                                              (str output-file)))]
+            (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
+                  foutput (fs/path (str finput ".ttf"))
+                  _       (bs/write-to-file! data finput)
+                  res     (sh/sh "fontforge" "-lang=ff" "-c"
+                                 (str/fmt "Open('%s'); Generate('%s')"
+                                          (str finput)
+                                          (str foutput)))]
               (when (zero? (:exit res))
-                (fs/slurp-bytes output-file))))
+                foutput)))
 
           (ttf-or-otf->woff [data]
-            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
-                  output-file (fs/path (str input-file ".woff"))
-                  _           (with-open [out (io/output-stream input-file)]
-                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
-                                (.flush ^OutputStream out))
-                  res         (sh/sh "sfnt2woff" (str input-file))]
+            ;; NOTE: foutput is not used directly, it represents the
+            ;; default output of the exection of the underlying
+            ;; command.
+            (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix "")
+                  foutput (fs/path (str finput ".woff"))
+                  _       (bs/write-to-file! data finput)
+                  res     (sh/sh "sfnt2woff" (str finput))]
               (when (zero? (:exit res))
-                (fs/slurp-bytes output-file))))
+                foutput)))
 
           (ttf-or-otf->woff2 [data]
-            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
-                  output-file (fs/path (str input-file ".woff2"))
-                  _           (with-open [out (io/output-stream input-file)]
-                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
-                                (.flush ^OutputStream out))
-                  res         (sh/sh "woff2_compress" (str input-file))]
+            ;; NOTE: foutput is not used directly, it represents the
+            ;; default output of the exection of the underlying
+            ;; command.
+            (let [finput  (tmp/tempfile :prefix "penpot.font." :suffix ".tmp")
+                  foutput (fs/path (str (fs/base finput) ".woff2"))
+                  _       (bs/write-to-file! data finput)
+                  res      (sh/sh "woff2_compress" (str finput))]
               (when (zero? (:exit res))
-                (fs/slurp-bytes output-file))))
+                foutput)))
 
           (woff->sfnt [data]
-            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
-                  _           (with-open [out (io/output-stream input-file)]
-                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
-                                (.flush ^OutputStream out))
-                  res         (sh/sh "woff2sfnt" (str input-file)
-                                     :out-enc :bytes)]
+            (let [finput  (tmp/tempfile :prefix "penpot" :suffix "")
+                  _       (bs/write-to-file! data finput)
+                  res     (sh/sh "woff2sfnt" (str finput)
+                                 :out-enc :bytes)]
               (when (zero? (:exit res))
                 (:out res))))
 
@@ -334,10 +308,11 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn configure-assets-storage
-  "Given storage map, returns a storage configured with the apropriate
-  backend for assets."
-  [storage conn]
-  (-> storage
-      (assoc :conn conn)
-      (assoc :backend (cf/get :assets-storage-backend :assets-fs))))
-
+  "Given storage map, returns a storage configured with the appropriate
+  backend for assets and optional connection attached."
+  ([storage]
+   (assoc storage :backend (cf/get :assets-storage-backend :assets-fs)))
+  ([storage conn]
+   (-> storage
+       (assoc :conn conn)
+       (assoc :backend (cf/get :assets-storage-backend :assets-fs)))))

backend/src/app/metrics.clj

@@ -5,48 +5,136 @@
 ;; Copyright (c) UXBOX Labs SL
 
 (ns app.metrics
+  (:refer-clojure :exclude [run!])
   (:require
-   [app.common.exceptions :as ex]
-   [app.util.logging :as l]
+   [app.common.logging :as l]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig])
   (:import
    io.prometheus.client.CollectorRegistry
    io.prometheus.client.Counter
+   io.prometheus.client.Counter$Child
    io.prometheus.client.Gauge
+   io.prometheus.client.Gauge$Child
    io.prometheus.client.Summary
+   io.prometheus.client.Summary$Child
+   io.prometheus.client.Summary$Builder
    io.prometheus.client.Histogram
+   io.prometheus.client.Histogram$Child
    io.prometheus.client.exporter.common.TextFormat
    io.prometheus.client.hotspot.DefaultExports
-   io.prometheus.client.jetty.JettyStatisticsCollector
-   org.eclipse.jetty.server.handler.StatisticsHandler
    java.io.StringWriter))
 
-(declare instrument-vars!)
-(declare instrument)
+(set! *warn-on-reflection* true)
+
 (declare create-registry)
 (declare create)
+(declare handler)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Entry Point
+;; METRICS SERVICE PROVIDER
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn- handler
-  [registry _request]
-  (let [samples  (.metricFamilySamples ^CollectorRegistry registry)
-        writer   (StringWriter.)]
-    (TextFormat/write004 writer samples)
-    {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
-     :body (.toString writer)}))
+(def default-metrics
+  {:update-file-changes
+   {:name "rpc_update_file_changes_total"
+    :help "A total number of changes submitted to update-file."
+    :type :counter}
 
-(s/def ::definitions
-  (s/map-of keyword? map?))
+   :update-file-bytes-processed
+   {:name "rpc_update_file_bytes_processed_total"
+    :help "A total number of bytes processed by update-file."
+    :type :counter}
 
-(defmethod ig/pre-init-spec ::metrics [_]
-  (s/keys :opt-un [::definitions]))
+   :rpc-mutation-timing
+   {:name "rpc_mutation_timing"
+    :help "RPC mutation method call timming."
+    :labels ["name"]
+    :type :histogram}
+
+   :rpc-command-timing
+   {:name "rpc_command_timing"
+    :help "RPC command method call timming."
+    :labels ["name"]
+    :type :histogram}
+
+   :rpc-query-timing
+   {:name "rpc_query_timing"
+    :help "RPC query method call timing."
+    :labels ["name"]
+    :type :histogram}
+
+   :websocket-active-connections
+   {:name "websocket_active_connections"
+    :help "Active websocket connections gauge"
+    :type :gauge}
+
+   :websocket-messages-total
+   {:name "websocket_message_total"
+    :help "Counter of processed messages."
+    :labels ["op"]
+    :type :counter}
+
+   :websocket-session-timing
+   {:name "websocket_session_timing"
+    :help "Websocket session timing (seconds)."
+    :type :summary}
+
+   :session-update-total
+   {:name "http_session_update_total"
+    :help "A counter of session update batch events."
+    :type :counter}
+
+   :tasks-timing
+   {:name "penpot_tasks_timing"
+    :help "Background tasks timing (milliseconds)."
+    :labels ["name"]
+    :type :summary}
+
+   :rlimit-queued-submissions
+   {:name "penpot_rlimit_queued_submissions"
+    :help "Current number of queued submissions on RLIMIT."
+    :labels ["name"]
+    :type :gauge}
+
+   :rlimit-used-permits
+   {:name "penpot_rlimit_used_permits"
+    :help "Current number of used permits on RLIMIT."
+    :labels ["name"]
+    :type :gauge}
+
+   :rlimit-acquires-total
+   {:name "penpot_rlimit_acquires_total"
+    :help "Total number of acquire operations on RLIMIT."
+    :labels ["name"]
+    :type :counter}
+
+   :executors-active-threads
+   {:name "penpot_executors_active_threads"
+    :help "Current number of threads available in the executor service."
+    :labels ["name"]
+    :type :gauge}
+
+   :executors-completed-tasks
+   {:name "penpot_executors_completed_tasks_total"
+    :help "Aproximate number of completed tasks by the executor."
+    :labels ["name"]
+    :type :counter}
+
+   :executors-running-threads
+   {:name "penpot_executors_running_threads"
+    :help "Current number of threads with state RUNNING."
+    :labels ["name"]
+    :type :gauge}
+
+   :executors-queued-submissions
+   {:name "penpot_executors_queued_submissions"
+    :help "Current number of queued submissions."
+    :labels ["name"]
+    :type :gauge}})
 
 (defmethod ig/init-key ::metrics
-  [_ {:keys [definitions] :as cfg}]
+  [_ _]
   (l/info :action "initialize metrics")
   (let [registry    (create-registry)
         definitions (reduce-kv (fn [res k v]
@@ -54,7 +142,7 @@
                                       (create)
                                       (assoc res k)))
                                {}
-                               definitions)]
+                               default-metrics)]
     {:handler (partial handler registry)
      :definitions definitions
      :registry registry}))
@@ -64,24 +152,45 @@
 (s/def ::metrics
   (s/keys :req-un [::registry ::handler]))
 
+(defn- handler
+  [registry _ respond _]
+  (let [samples  (.metricFamilySamples ^CollectorRegistry registry)
+        writer   (StringWriter.)]
+    (TextFormat/write004 writer samples)
+    (respond {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
+              :body (.toString writer)})))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Implementation
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(def default-empty-labels (into-array String []))
+
+(def default-quantiles
+  [[0.5  0.01]
+   [0.90 0.01]
+   [0.99 0.001]])
+
+(def default-histogram-buckets
+  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+
+(defn run!
+  [{:keys [definitions]} {:keys [id] :as params}]
+  (when-let [mobj (get definitions id)]
+    ((::fn mobj) params)
+    true))
+
 (defn create-registry
   []
   (let [registry (CollectorRegistry.)]
     (DefaultExports/register registry)
     registry))
 
-(defmacro with-measure
-  [& {:keys [expr cb]}]
-  `(let [start# (System/nanoTime)
-         tdown# ~cb]
-     (try
-       ~expr
-       (finally
-         (tdown# (/ (- (System/nanoTime) start#) 1000000))))))
+(defn- is-array?
+  [o]
+  (let [oc (class o)]
+    (and (.isArray ^Class oc)
+         (= (.getComponentType oc) String))))
 
 (defn make-counter
   [{:keys [name help registry reg labels] :as props}]
@@ -92,18 +201,11 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (.inc ^Counter instance))
-
-      (invoke [_ cmd labels]
-        (.. ^Counter instance
-            (labels (into-array String labels))
-            (inc))))))
+    {::instance instance
+     ::fn (fn [{:keys [inc labels] :or {inc 1 labels default-empty-labels}}]
+            (let [instance (.labels instance (if (is-array? labels) labels (into-array String labels)))]
+              (.inc ^Counter$Child instance (double inc))))}))
 
 (defn make-gauge
   [{:keys [name help registry reg labels] :as props}]
@@ -114,57 +216,33 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
-
-      clojure.lang.IFn
-      (invoke [_ cmd]
-        (case cmd
-          :inc (.inc ^Gauge instance)
-          :dec (.dec ^Gauge instance)))
-
-      (invoke [_ cmd labels]
-        (let [labels (into-array String [labels])]
-          (case cmd
-            :inc (.. ^Gauge instance (labels labels) (inc))
-            :dec (.. ^Gauge instance (labels labels) (dec))))))))
-
-(def default-quantiles
-  [[0.75 0.02]
-   [0.99 0.001]])
+    {::instance instance
+     ::fn (fn [{:keys [inc dec labels val] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Gauge instance (if (is-array? labels) labels (into-array String labels)))]
+              (cond (number? inc) (.inc ^Gauge$Child instance (double inc))
+                    (number? dec) (.dec ^Gauge$Child instance (double dec))
+                    (number? val) (.set ^Gauge$Child instance (double val)))))}))
 
 (defn make-summary
   [{:keys [name help registry reg labels max-age quantiles buckets]
-    :or {max-age 3600 buckets 6 quantiles default-quantiles} :as props}]
+    :or {max-age 3600 buckets 12 quantiles default-quantiles} :as props}]
   (let [registry (or registry reg)
-        instance (doto (Summary/build)
+        builder  (doto (Summary/build)
                    (.name name)
                    (.help help))
         _        (when (seq quantiles)
-                   (.maxAgeSeconds ^Summary instance max-age)
-                   (.ageBuckets ^Summary instance buckets))
+                   (.maxAgeSeconds ^Summary$Builder builder ^long max-age)
+                   (.ageBuckets ^Summary$Builder builder buckets))
         _        (doseq [[q e] quantiles]
-                   (.quantile ^Summary instance q e))
+                   (.quantile ^Summary$Builder builder q e))
         _        (when (seq labels)
-                   (.labelNames instance (into-array String labels)))
-        instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
+                   (.labelNames ^Summary$Builder builder (into-array String labels)))
+        instance (.register ^Summary$Builder builder registry)]
 
-      clojure.lang.IFn
-      (invoke [_ cmd val]
-        (.observe ^Summary instance val))
-
-      (invoke [_ cmd val labels]
-        (.. ^Summary instance
-            (labels (into-array String labels))
-            (observe val))))))
-
-(def default-histogram-buckets
-  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+    {::instance instance
+     ::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Summary instance (if (is-array? labels) labels (into-array String labels)))]
+              (.observe ^Summary$Child instance val)))}))
 
 (defn make-histogram
   [{:keys [name help registry reg labels buckets]
@@ -177,18 +255,11 @@
         _        (when (seq labels)
                    (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
-    (reify
-      clojure.lang.IDeref
-      (deref [_] instance)
 
-      clojure.lang.IFn
-      (invoke [_ cmd val]
-        (.observe ^Histogram instance val))
-
-      (invoke [_ cmd val labels]
-        (.. ^Histogram instance
-            (labels (into-array String labels))
-            (observe val))))))
+    {::instance instance
+     ::fn (fn [{:keys [val labels] :or {labels default-empty-labels}}]
+            (let [instance (.labels ^Histogram instance (if (is-array? labels) labels (into-array String labels)))]
+              (.observe ^Histogram$Child instance val)))}))
 
 (defn create
   [{:keys [type] :as props}]
@@ -197,118 +268,3 @@
     :gauge   (make-gauge props)
     :summary (make-summary props)
     :histogram (make-histogram props)))
-
-(defn wrap-counter
-  ([rootf mobj]
-   (let [mdata (meta rootf)
-         origf (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-          (mobj :inc)
-          (origf a))
-         ([a b]
-          (mobj :inc)
-          (origf a b))
-         ([a b c]
-          (mobj :inc)
-          (origf a b c))
-         ([a b c d]
-          (mobj :inc)
-          (origf a b c d))
-         ([a b c d & more]
-          (mobj :inc)
-          (apply origf a b c d more)))
-       (assoc mdata ::original origf))))
-  ([rootf mobj labels]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-          (mobj :inc labels)
-          (origf a))
-         ([a b]
-          (mobj :inc labels)
-          (origf a b))
-         ([a b & more]
-          (mobj :inc labels)
-          (apply origf a b more)))
-       (assoc mdata ::original origf)))))
-
-(defn wrap-summary
-  ([rootf mobj]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-         (with-measure
-           :expr (origf a)
-           :cb   #(mobj :observe %)))
-         ([a b]
-          (with-measure
-            :expr (origf a b)
-            :cb   #(mobj :observe %)))
-         ([a b & more]
-          (with-measure
-            :expr (apply origf a b more)
-            :cb   #(mobj :observe %))))
-       (assoc mdata ::original origf))))
-
-  ([rootf mobj labels]
-   (let [mdata  (meta rootf)
-         origf  (::original mdata rootf)]
-     (with-meta
-       (fn
-         ([a]
-         (with-measure
-           :expr (origf a)
-           :cb   #(mobj :observe % labels)))
-         ([a b]
-          (with-measure
-            :expr (origf a b)
-            :cb   #(mobj :observe % labels)))
-         ([a b & more]
-          (with-measure
-            :expr (apply origf a b more)
-            :cb   #(mobj :observe % labels))))
-       (assoc mdata ::original origf)))))
-
-(defn instrument-vars!
-  [vars {:keys [wrap] :as props}]
-  (let [obj (create props)]
-    (cond
-      (instance? Counter @obj)
-      (doseq [var vars]
-        (alter-var-root var (or wrap wrap-counter) obj))
-
-      (instance? Summary @obj)
-      (doseq [var vars]
-        (alter-var-root var (or wrap wrap-summary) obj))
-
-      :else
-      (ex/raise :type :not-implemented))))
-
-(defn instrument
-  [f {:keys [wrap] :as props}]
-  (let [obj (create props)]
-    (cond
-      (instance? Counter @obj)
-      ((or wrap wrap-counter) f obj)
-
-      (instance? Summary @obj)
-      ((or wrap wrap-summary) f obj)
-
-      (instance? Histogram @obj)
-      ((or wrap wrap-summary) f obj)
-
-      :else
-      (ex/raise :type :not-implemented))))
-
-(defn instrument-jetty!
-  [^CollectorRegistry registry ^StatisticsHandler handler]
-  (doto (JettyStatisticsCollector. handler)
-    (.register registry))
-  nil)
-

backend/src/app/migrations.clj

@@ -6,7 +6,7 @@
 
 (ns app.migrations
   (:require
-   [app.migrations.migration-0023 :as mg0023]
+   [app.migrations.clj.migration-0023 :as mg0023]
    [app.util.migrations :as mg]
    [integrant.core :as ig]))
 
@@ -193,6 +193,57 @@
 
    {:name "0061-mod-file-table"
     :fn (mg/resource "app/migrations/sql/0061-mod-file-table.sql")}
+
+   {:name "0062-fix-metadata-media"
+    :fn (mg/resource "app/migrations/sql/0062-fix-metadata-media.sql")}
+
+   {:name "0063-add-share-link-table"
+    :fn (mg/resource "app/migrations/sql/0063-add-share-link-table.sql")}
+
+   {:name "0064-mod-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0064-mod-audit-log-table.sql")}
+
+   {:name "0065-add-trivial-spelling-fixes"
+    :fn (mg/resource "app/migrations/sql/0065-add-trivial-spelling-fixes.sql")}
+
+   {:name "0066-add-frame-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0066-add-frame-thumbnail-table.sql")}
+
+   {:name "0067-add-team-invitation-table"
+    :fn (mg/resource "app/migrations/sql/0067-add-team-invitation-table.sql")}
+
+   {:name "0068-mod-storage-object-table"
+    :fn (mg/resource "app/migrations/sql/0068-mod-storage-object-table.sql")}
+
+   {:name "0069-add-file-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0069-add-file-thumbnail-table.sql")}
+
+   {:name "0070-del-frame-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0070-del-frame-thumbnail-table.sql")}
+
+   {:name "0071-add-file-object-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0071-add-file-object-thumbnail-table.sql")}
+
+   {:name "0072-mod-file-object-thumbnail-table"
+    :fn (mg/resource "app/migrations/sql/0072-mod-file-object-thumbnail-table.sql")}
+
+   {:name "0073-mod-file-media-object-constraints"
+    :fn (mg/resource "app/migrations/sql/0073-mod-file-media-object-constraints.sql")}
+
+   {:name "0074-mod-file-library-rel-constraints"
+    :fn (mg/resource "app/migrations/sql/0074-mod-file-library-rel-constraints.sql")}
+
+   {:name "0075-mod-share-link-table"
+    :fn (mg/resource "app/migrations/sql/0075-mod-share-link-table.sql")}
+
+   {:name "0076-mod-storage-object-table"
+    :fn (mg/resource "app/migrations/sql/0076-mod-storage-object-table.sql")}
+
+   {:name "0077-mod-comment-thread-table"
+    :fn (mg/resource "app/migrations/sql/0077-mod-comment-thread-table.sql")}
+
+   {:name "0078-mod-file-media-object-table-drop-cascade"
+    :fn (mg/resource "app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql")}
    ])
 
 

backend/src/app/migrations/clj/migration_0023.clj

@@ -4,7 +4,7 @@
 ;;
 ;; Copyright (c) UXBOX Labs SL
 
-(ns app.migrations.migration-0023
+(ns app.migrations.clj.migration-0023
   (:require
    [app.db :as db]
    [app.util.blob :as blob]))

backend/src/app/migrations/sql/0035-add-storage-tables.sql

@@ -22,7 +22,7 @@ CREATE TABLE storage_data (
 CREATE INDEX storage_data__id__idx ON storage_data(id);
 
 -- Table used for store inflight upload ids, for later recheck and
--- delete possible staled files that exists on the phisical storage
+-- delete possible staled files that exists on the physical storage
 -- but does not exists in the 'storage_object' table.
 
 CREATE TABLE storage_pending (

backend/src/app/migrations/sql/0062-fix-metadata-media.sql

@@ -1,4 +1,4 @@
--- Fix problem with content-type inconherence
+-- Fix problem with content-type incoherence
 
 UPDATE storage_object so
 SET metadata = jsonb_set(metadata, '{~:content-type}', to_jsonb(fmo.mtype))

backend/src/app/migrations/sql/0063-add-share-link-table.sql

@@ -0,0 +1,12 @@
+CREATE TABLE share_link (
+  id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE,
+  owner_id uuid NULL REFERENCES profile(id) ON DELETE SET NULL DEFERRABLE,
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  pages uuid[],
+  flags text[]
+);
+
+CREATE INDEX share_link_file_id_idx ON share_link(file_id);
+CREATE INDEX share_link_owner_id_idx ON share_link(owner_id);

backend/src/app/migrations/sql/0064-mod-audit-log-table.sql

@@ -0,0 +1,13 @@
+ALTER TABLE audit_log
+  ADD COLUMN tracked_at timestamptz NULL DEFAULT clock_timestamp(),
+  ADD COLUMN source text NULL,
+  ADD COLUMN context jsonb NULL;
+
+ALTER TABLE audit_log
+  ALTER COLUMN source SET STORAGE external,
+  ALTER COLUMN context SET STORAGE external;
+
+UPDATE audit_log SET source = 'backend', tracked_at=created_at;
+
+-- ALTER TABLE audit_log ALTER COLUMN source SET NOT NULL;
+-- ALTER TABLE audit_log ALTER COLUMN tracked_at SET NOT NULL;

backend/src/app/migrations/sql/0065-add-trivial-spelling-fixes.sql

@@ -0,0 +1,2 @@
+ALTER INDEX file__modified_at__has_media_trimed__idx RENAME TO file__modified_at__has_media_trimmed__idx;
+ALTER INDEX media_bject__file_id__idx RENAME TO media_object__file_id__idx;

backend/src/app/migrations/sql/0066-add-frame-thumbnail-table.sql

@@ -0,0 +1,13 @@
+CREATE TABLE file_frame_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
+  frame_id uuid NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+
+  data text NULL,
+
+  PRIMARY KEY(file_id, frame_id)
+);
+
+ALTER TABLE file_frame_thumbnail
+  ALTER COLUMN data SET STORAGE external;

backend/src/app/migrations/sql/0067-add-team-invitation-table.sql

@@ -0,0 +1,14 @@
+CREATE TABLE team_invitation (  
+  team_id uuid NOT NULL REFERENCES team(id) ON DELETE CASCADE,
+  email_to text NOT NULL,
+  role text NOT NULL,
+  valid_until timestamptz NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT now(),
+
+  PRIMARY KEY(team_id, email_to)
+);
+
+ALTER TABLE team_invitation
+  ALTER COLUMN email_to SET STORAGE external,
+  ALTER COLUMN role SET STORAGE external;

backend/src/app/migrations/sql/0068-mod-storage-object-table.sql

@@ -0,0 +1,3 @@
+CREATE INDEX storage_object__hash_backend_bucket__idx
+    ON storage_object ((metadata->>'~:hash'), (metadata->>'~:bucket'), backend)
+ WHERE deleted_at IS NULL;

backend/src/app/migrations/sql/0069-add-file-thumbnail-table.sql

@@ -0,0 +1,14 @@
+CREATE TABLE file_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
+  revn bigint NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT now(),
+  deleted_at timestamptz NULL,
+  data text NULL,
+  props jsonb NULL,
+  PRIMARY KEY(file_id, revn)
+);
+
+ALTER TABLE file_thumbnail
+  ALTER COLUMN data SET STORAGE external,
+  ALTER COLUMN props SET STORAGE external;

backend/src/app/migrations/sql/0070-del-frame-thumbnail-table.sql

@@ -0,0 +1 @@
+DROP TABLE file_frame_thumbnail;

backend/src/app/migrations/sql/0071-add-file-object-thumbnail-table.sql

@@ -0,0 +1,11 @@
+CREATE TABLE file_object_thumbnail (
+  file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,
+  object_id uuid NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  data text NULL,
+
+  PRIMARY KEY(file_id, object_id)
+);
+
+ALTER TABLE file_object_thumbnail
+  ALTER COLUMN data SET STORAGE external;

backend/src/app/migrations/sql/0072-mod-file-object-thumbnail-table.sql

@@ -0,0 +1,4 @@
+TRUNCATE TABLE file_object_thumbnail;
+
+ALTER TABLE file_object_thumbnail
+  ALTER COLUMN object_id TYPE text;

backend/src/app/migrations/sql/0073-mod-file-media-object-constraints.sql

@@ -0,0 +1,11 @@
+ALTER TABLE file_media_object
+ALTER CONSTRAINT file_media_object_media_id_fkey DEFERRABLE INITIALLY IMMEDIATE;
+
+ALTER TABLE file_media_object
+ALTER CONSTRAINT file_media_object_thumbnail_id_fkey DEFERRABLE INITIALLY IMMEDIATE;
+
+ALTER TABLE file_media_object
+RENAME CONSTRAINT media_object_file_id_fkey TO file_media_object_file_id_fkey;
+
+ALTER TABLE file_media_object
+ALTER CONSTRAINT file_media_object_file_id_fkey DEFERRABLE INITIALLY IMMEDIATE;

backend/src/app/migrations/sql/0074-mod-file-library-rel-constraints.sql

@@ -0,0 +1,5 @@
+ALTER TABLE file_library_rel
+ALTER CONSTRAINT file_library_rel_file_id_fkey DEFERRABLE INITIALLY IMMEDIATE;
+
+ALTER TABLE file_library_rel
+ALTER CONSTRAINT file_library_rel_library_file_id_fkey DEFERRABLE INITIALLY IMMEDIATE;

backend/src/app/migrations/sql/0075-mod-share-link-table.sql

@@ -0,0 +1,5 @@
+ALTER TABLE share_link
+  ADD COLUMN who_comment text NOT NULL DEFAULT('team'),
+  ADD COLUMN who_inspect text NOT NULL DEFAULT('team');
+
+--- TODO: remove flags column in 1.15.x

backend/src/app/migrations/sql/0076-mod-storage-object-table.sql

@@ -0,0 +1,10 @@
+-- Renames the old, already deprecated backend name with new one on
+-- all storage object rows.
+
+UPDATE storage_object
+   SET backend = 'assets-fs'
+ WHERE backend = 'fs';
+
+UPDATE storage_object
+   SET backend = 'assets-s3'
+ WHERE backend = 's3';

backend/src/app/migrations/sql/0077-mod-comment-thread-table.sql

@@ -0,0 +1,3 @@
+--- Add frame_id field.
+ALTER TABLE comment_thread
+  ADD COLUMN frame_id uuid NULL DEFAULT '00000000-0000-0000-0000-000000000000';

backend/src/app/migrations/sql/0078-mod-file-media-object-table-drop-cascade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE file_media_object
+ DROP CONSTRAINT file_media_object_media_id_fkey,
+  ADD CONSTRAINT file_media_object_media_id_fkey
+         FOREIGN KEY (media_id) REFERENCES storage_object(id) ON DELETE NO ACTION DEFERRABLE;
+
+ALTER TABLE file_media_object
+ DROP CONSTRAINT file_media_object_thumbnail_id_fkey,
+  ADD CONSTRAINT file_media_object_thumbnail_id_fkey
+         FOREIGN KEY (thumbnail_id) REFERENCES storage_object(id) ON DELETE NO ACTION DEFERRABLE;

backend/src/app/migrations/sql/XXXX-drop-obsolete-tables.sql

@@ -1,5 +1,5 @@
 --- This is a second migration but it should be applied when manual
---- migration intervention is alteady executed.
+--- migration intervention is already executed.
 
 ALTER TABLE file_media_object ALTER COLUMN media_id SET NOT NULL;
 DROP TABLE file_media_thumbnail;

backend/src/app/msgbus.clj

@@ -7,18 +7,20 @@
 (ns app.msgbus
   "The msgbus abstraction implemented using redis as underlying backend."
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
+   [app.common.transit :as t]
    [app.config :as cfg]
-   [app.util.blob :as blob]
-   [app.util.logging :as l]
+   [app.util.async :as aa]
    [app.util.time :as dt]
+   [app.worker :as wrk]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]
    [promesa.core :as p])
   (:import
-   java.time.Duration
    io.lettuce.core.RedisClient
    io.lettuce.core.RedisURI
    io.lettuce.core.api.StatefulConnection
@@ -29,7 +31,12 @@
    io.lettuce.core.codec.StringCodec
    io.lettuce.core.pubsub.RedisPubSubListener
    io.lettuce.core.pubsub.StatefulRedisPubSubConnection
-   io.lettuce.core.pubsub.api.async.RedisPubSubAsyncCommands))
+   io.lettuce.core.pubsub.api.sync.RedisPubSubCommands
+   io.lettuce.core.resource.ClientResources
+   io.lettuce.core.resource.DefaultClientResources
+   java.time.Duration))
+
+(set! *warn-on-reflection* true)
 
 (def ^:private prefix (cfg/get :tenant))
 
@@ -37,266 +44,253 @@
   [topic]
   (str prefix "." topic))
 
-(def xform-prefix (map prefix-topic))
-(def xform-topics (map (fn [m] (update m :topics #(into #{} xform-prefix %)))))
-(def xform-topic  (map (fn [m] (update m :topic prefix-topic))))
+(def ^:private xform-prefix-topic
+  (map (fn [obj] (update obj :topic prefix-topic))))
 
-(s/def ::redis-uri ::us/string)
-(s/def ::buffer-size ::us/integer)
-
-(defmulti init-backend :backend)
-(defmulti stop-backend :backend)
-(defmulti init-pub-loop :backend)
-(defmulti init-sub-loop :backend)
-
-(defmethod ig/pre-init-spec ::msgbus [_]
-  (s/keys :opt-un [::buffer-size ::redis-uri]))
+(declare ^:private redis-connect)
+(declare ^:private redis-disconnect)
+(declare ^:private start-io-loop)
+(declare ^:private subscribe)
+(declare ^:private purge)
+(declare ^:private redis-pub)
+(declare ^:private redis-sub)
+(declare ^:private redis-unsub)
 
 (defmethod ig/prep-key ::msgbus
   [_ cfg]
-  (merge {:buffer-size 128} cfg))
+  (merge {:buffer-size 128
+          :timeout (dt/duration {:seconds 30})}
+         (d/without-nils cfg)))
+
+(s/def ::timeout ::dt/duration)
+(s/def ::redis-uri ::us/string)
+(s/def ::buffer-size ::us/integer)
+
+(defmethod ig/pre-init-spec ::msgbus [_]
+  (s/keys :req-un [::buffer-size ::redis-uri ::timeout ::wrk/executor]))
 
 (defmethod ig/init-key ::msgbus
-  [_ {:keys [backend buffer-size] :as cfg}]
-  (l/debug :action "initialize msgbus"
-           :backend (name backend))
-  (let [cfg     (init-backend cfg)
+  [_ {:keys [buffer-size redis-uri] :as cfg}]
+  (l/info :hint "initialize msgbus"
+          :buffer-size buffer-size
+          :redis-uri redis-uri)
+  (let [cmd-ch (a/chan buffer-size)
+        rcv-ch (a/chan (a/dropping-buffer buffer-size))
+        pub-ch (a/chan (a/dropping-buffer buffer-size) xform-prefix-topic)
+        state  (agent {} :error-handler #(l/error :cause % :hint "unexpected error on agent" ::l/async false))
+        cfg    (-> (redis-connect cfg)
+                   (assoc ::cmd-ch cmd-ch)
+                   (assoc ::rcv-ch rcv-ch)
+                   (assoc ::pub-ch pub-ch)
+                   (assoc ::state state))]
 
-        ;; Channel used for receive publications from the application.
-        pub-ch  (-> (a/dropping-buffer buffer-size)
-                    (a/chan xform-topic))
-
-        ;; Channel used for receive subscription requests.
-        sub-ch  (a/chan 1 xform-topics)
-
-        cfg     (-> cfg
-                    (assoc ::pub-ch pub-ch)
-                    (assoc ::sub-ch sub-ch))]
-
-    (init-pub-loop cfg)
-    (init-sub-loop cfg)
+    (start-io-loop cfg)
 
     (with-meta
-      (fn run
-        ([command] (run command nil))
-        ([command params]
-         (a/go
-           (case command
-             :pub (a/>! pub-ch params)
-             :sub (a/>! sub-ch params)))))
+      (fn [& {:keys [cmd] :as params}]
+        (a/go
+          (case cmd
+            :pub   (a/>! pub-ch params)
+            :sub   (a/<! (subscribe cfg params))
+            :purge (a/<! (purge cfg params))
+            (l/error :hint "unexpeced error on msgbus command processing" :params params))))
       cfg)))
 
 (defmethod ig/halt-key! ::msgbus
   [_ f]
   (let [mdata (meta f)]
-    (stop-backend mdata)
-    (a/close! (::pub-ch mdata))
-    (a/close! (::sub-ch mdata))))
+    (redis-disconnect mdata)
+    (a/close! (::cmd-ch mdata))
+    (a/close! (::rcv-ch mdata))))
 
-;; --- IN-MEMORY BACKEND IMPL
+;; --- IMPL
 
-(defmethod init-backend :memory [cfg] cfg)
-(defmethod stop-backend :memory [_])
-(defmethod init-pub-loop :memory [_])
+(defn- redis-connect
+  [{:keys [redis-uri timeout] :as cfg}]
+  (let [codec     (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE)
 
-(defmethod init-sub-loop :memory
-  [{:keys [::sub-ch ::pub-ch]}]
-  (a/go-loop [state {}]
-    (let [[val port] (a/alts! [pub-ch sub-ch])]
-      (cond
-        (and (= port sub-ch) (some? val))
-        (let [{:keys [topics chan]} val]
-          (recur (reduce #(update %1 %2 (fnil conj #{}) chan) state topics)))
+        resources (.. (DefaultClientResources/builder)
+                      (ioThreadPoolSize 4)
+                      (computationThreadPoolSize 4)
+                      (build))
 
-        (and (= port pub-ch) (some? val))
-        (let [topic   (:topic val)
-              message (:message val)
-              state   (loop [state state
-                             chans (get state topic)]
-                        (if-let [c (first chans)]
-                          (if (a/>! c message)
-                            (recur state (rest chans))
-                            (recur (update state topic disj c)
-                                   (rest chans)))
-                          state))]
-          (recur state))
+        uri       (RedisURI/create redis-uri)
+        rclient   (RedisClient/create ^ClientResources resources ^RedisURI uri)
 
-        :else
-        (->> (vals state)
-             (mapcat identity)
-             (run! a/close!))))))
+        pconn     (.connect ^RedisClient rclient ^RedisCodec codec)
+        sconn     (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
 
-
-;; Add a unique listener to connection
-
-;; --- REDIS BACKEND IMPL
-
-(declare impl-redis-open?)
-(declare impl-redis-pub)
-(declare impl-redis-sub)
-(declare impl-redis-unsub)
-
-(defmethod init-backend :redis
-  [{:keys [redis-uri] :as cfg}]
-  (let [codec    (RedisCodec/of StringCodec/UTF8 ByteArrayCodec/INSTANCE)
-
-        uri      (RedisURI/create redis-uri)
-        rclient  (RedisClient/create ^RedisURI uri)
-
-        pub-conn (.connect ^RedisClient rclient ^RedisCodec codec)
-        sub-conn (.connectPubSub ^RedisClient rclient ^RedisCodec codec)]
-
-    (.setTimeout ^StatefulRedisConnection pub-conn ^Duration (dt/duration {:seconds 10}))
-    (.setTimeout ^StatefulRedisPubSubConnection sub-conn ^Duration (dt/duration {:seconds 10}))
+    (.setTimeout ^StatefulRedisConnection pconn ^Duration timeout)
+    (.setTimeout ^StatefulRedisPubSubConnection sconn ^Duration timeout)
 
     (-> cfg
-        (assoc ::pub-conn pub-conn)
-        (assoc ::sub-conn sub-conn))))
+        (assoc ::resources resources)
+        (assoc ::pconn pconn)
+        (assoc ::sconn sconn))))
 
-(defmethod stop-backend :redis
-  [{:keys [::pub-conn ::sub-conn] :as cfg}]
-  (.close ^StatefulRedisConnection pub-conn)
-  (.close ^StatefulRedisPubSubConnection sub-conn))
+(defn- redis-disconnect
+  [{:keys [::pconn ::sconn ::resources] :as cfg}]
+  (.. ^StatefulConnection pconn close)
+  (.. ^StatefulConnection sconn close)
+  (.shutdown ^ClientResources resources))
 
-(defmethod init-pub-loop :redis
-  [{:keys [::pub-conn ::pub-ch]}]
-  (let [rac (.async ^StatefulRedisConnection pub-conn)]
-    (a/go-loop []
-      (when-let [val (a/<! pub-ch)]
-        (let [result (a/<! (impl-redis-pub rac val))]
-          (when (and (impl-redis-open? pub-conn)
-                     (ex/exception? result))
-            (l/error :cause result
-                     :hint "unexpected error on publish message to redis")))
-        (recur)))))
+(defn- conj-subscription
+  "A low level function that is responsible to create on-demand
+  subscriptions on redis. It reuses the same subscription if it is
+  already established. Intended to be executed in agent."
+  [nsubs cfg topic chan]
+  (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
+    (when (= 1 (count nsubs))
+      (l/trace :hint "open subscription" :topic topic ::l/async false)
+      (redis-sub cfg topic))
+    nsubs))
 
-(defmethod init-sub-loop :redis
-  [{:keys [::sub-conn ::sub-ch buffer-size]}]
-  (let [rcv-ch  (a/chan (a/dropping-buffer buffer-size))
-        chans   (agent {} :error-handler #(l/error :cause % :hint "unexpected error on agent"))
-        rac     (.async ^StatefulRedisPubSubConnection sub-conn)]
+(defn- disj-subscription
+  "A low level function responsible on removing subscriptions. The
+  subscription is trully removed from redis once no single local
+  subscription is look for it. Intended to be executed in agent."
+  [nsubs cfg topic chan]
+  (let [nsubs (disj nsubs chan)]
+    (when (empty? nsubs)
+      (l/trace :hint "close subscription" :topic topic ::l/async false)
+      (redis-unsub cfg topic))
+    nsubs))
 
-    ;; Add a unique listener to connection
-    (.addListener sub-conn
-                  (reify RedisPubSubListener
-                    (message [it pattern topic message])
-                    (message [it topic message]
-                      ;; There are no back pressure, so we use a slidding
-                      ;; buffer for cases when the pubsub broker sends
-                      ;; more messages that we can process.
-                      (let [val {:topic topic :message (blob/decode message)}]
-                        (when-not (a/offer! rcv-ch val)
-                          (l/warn :msg "dropping message on subscription loop"))))
-                    (psubscribed [it pattern count])
-                    (punsubscribed [it pattern count])
-                    (subscribed [it topic count])
-                    (unsubscribed [it topic count])))
+(defn- subscribe-to-topics
+  "Function responsible to attach local subscription to the
+  state. Intended to be used in agent."
+  [state cfg topics chan done-ch]
+  (aa/with-closing done-ch
+    (let [state (update state :chans assoc chan topics)]
+      (reduce (fn [state topic]
+                (update-in state [:topics topic] conj-subscription cfg topic chan))
+              state
+              topics))))
 
-    (letfn [(subscribe-to-single-topic [nsubs topic chan]
-              (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
-                (when (= 1 (count nsubs))
-                  (let [result (a/<!! (impl-redis-sub rac topic))]
-                    (l/trace :action "open subscription"
-                             :topic topic)
-                    (when (ex/exception? result)
-                      (l/error :cause result
-                               :hint "unexpected exception on subscribing"
-                               :topic topic))))
-                nsubs))
+(defn- unsubscribe-single-channel
+  "Auxiliar function responsible on removing a single local
+  subscription from the state."
+  [state cfg chan]
+  (let [topics (get-in state [:chans chan])
+        state  (update state :chans dissoc chan)]
+    (reduce (fn [state topic]
+              (update-in state [:topics topic] disj-subscription cfg topic chan))
+            state
+            topics)))
 
-            (subscribe-to-topics [state topics chan]
-              (let [state  (update state :chans assoc chan topics)]
-                (reduce (fn [state topic]
-                          (update-in state [:topics topic] subscribe-to-single-topic topic chan))
-                        state
-                        topics)))
-
-            (unsubscribe-from-single-topic [nsubs topic chan]
-              (let [nsubs (disj nsubs chan)]
-                (when (empty? nsubs)
-                  (let [result (a/<!! (impl-redis-unsub rac topic))]
-                    (l/trace :action "close subscription"
-                             :topic topic)
-                    (when (and (impl-redis-open? sub-conn)
-                               (ex/exception? result))
-                      (l/error :cause result
-                               :hint "unexpected exception on unsubscribing"
-                               :topic topic))))
-                nsubs))
-
-            (unsubscribe-channels [state pending]
-              (reduce (fn [state ch]
-                        (let [topics (get-in state [:chans ch])
-                              state  (update state :chans dissoc ch)]
-                          (reduce (fn [state topic]
-                                    (update-in state [:topics topic] unsubscribe-from-single-topic topic ch))
-                                  state
-                                  topics)))
-                      state
-                      pending))]
-
-      ;; Asynchronous subscription loop;
-      (a/go-loop []
-        (if-let [{:keys [topics chan]} (a/<! sub-ch)]
-          (do
-            (send-off chans subscribe-to-topics topics chan)
-            (recur))
-          (a/close! rcv-ch)))
-
-      ;; Asyncrhonous message processing loop;x
-      (a/go-loop []
-        (if-let [{:keys [topic message]} (a/<! rcv-ch)]
-          ;; This means we receive data from redis and we need to
-          ;; forward it to the underlying subscriptions.
-          (let [pending (loop [chans   (seq (get-in @chans [:topics topic]))
-                               pending #{}]
-                          (if-let [ch (first chans)]
-                            (if (a/>! ch message)
-                              (recur (rest chans) pending)
-                              (recur (rest chans) (conj pending ch)))
-                            pending))]
-            (some->> (seq pending)
-                     (send-off chans unsubscribe-channels))
-
-            (recur))
-
-          ;; Stop condition; close all underlying subscriptions and
-          ;; exit. The close operation is performed asynchronously.
-          (send-off chans (fn [state]
-                            (->> (vals state)
-                                 (mapcat identity)
-                                 (filter some?)
-                                 (run! a/close!)))))))))
+(defn- unsubscribe-channels
+  "Function responsible from detach from state a seq of channels,
+  useful when client disconnects or in-bulk unsubscribe
+  operations. Intended to be executed in agent."
+  [state cfg channels done-ch]
+  (aa/with-closing done-ch
+    (reduce #(unsubscribe-single-channel %1 cfg %2) state channels)))
 
 
-(defn- impl-redis-open?
-  [^StatefulConnection conn]
-  (.isOpen conn))
+(defn- subscribe
+  [{:keys [::state executor] :as cfg} {:keys [topic topics chan]}]
+  (let [done-ch (a/chan)
+        topics  (into [] (map prefix-topic) (if topic [topic] topics))]
+    (l/debug :hint "subscribe" :topics topics)
+    (send-via executor state subscribe-to-topics cfg topics chan done-ch)
+    done-ch))
 
-(defn- impl-redis-pub
-  [^RedisAsyncCommands rac {:keys [topic message]}]
-  (let [message (blob/encode message)
-        res     (a/chan 1)]
-    (-> (.publish rac ^String topic ^bytes message)
-        (p/finally (fn [_ e]
-                     (when e (a/>!! res e))
+(defn- purge
+  [{:keys [::state executor] :as cfg} {:keys [chans]}]
+  (l/trace :hint "purge" :chans (count chans))
+  (let [done-ch (a/chan)]
+    (send-via executor state unsubscribe-channels cfg chans done-ch)
+    done-ch))
+
+(defn- create-listener
+  [rcv-ch]
+  (reify RedisPubSubListener
+    (message [_ _pattern _topic _message])
+    (message [_ topic message]
+      ;; There are no back pressure, so we use a slidding
+      ;; buffer for cases when the pubsub broker sends
+      ;; more messages that we can process.
+      (let [val {:topic topic :message (t/decode message)}]
+        (when-not (a/offer! rcv-ch val)
+          (l/warn :msg "dropping message on subscription loop"))))
+    (psubscribed [_ _pattern _count])
+    (punsubscribed [_ _pattern _count])
+    (subscribed [_ _topic _count])
+    (unsubscribed [_ _topic _count])))
+
+(defn start-io-loop
+  [{:keys [::sconn ::rcv-ch ::pub-ch ::state executor] :as cfg}]
+
+  ;; Add a single listener to the pubsub connection
+  (.addListener ^StatefulRedisPubSubConnection sconn
+                ^RedisPubSubListener (create-listener rcv-ch))
+
+  (letfn [(send-to-topic [topic message]
+            (a/go-loop [chans  (seq (get-in @state [:topics topic]))
+                        closed #{}]
+              (if-let [ch (first chans)]
+                (if (a/>! ch message)
+                  (recur (rest chans) closed)
+                  (recur (rest chans) (conj closed ch)))
+                (seq closed))))
+
+          (process-incoming [{:keys [topic message]}]
+            (a/go
+              (when-let [closed (a/<! (send-to-topic topic message))]
+                (send-via executor state unsubscribe-channels cfg closed nil))))
+          ]
+
+  (a/go-loop []
+    (let [[val port] (a/alts! [pub-ch rcv-ch])]
+      (cond
+        (nil? val)
+        (do
+          (l/trace :hint "stoping io-loop, nil received")
+          (send-via executor state (fn [state]
+                                     (->> (vals state)
+                                          (mapcat identity)
+                                          (filter some?)
+                                          (run! a/close!))
+                                     nil)))
+
+        (= port rcv-ch)
+        (do
+          (a/<! (process-incoming val))
+          (recur))
+
+        (= port pub-ch)
+        (let [result (a/<! (redis-pub cfg val))]
+          (when (ex/exception? result)
+            (l/error :hint "unexpected error on publishing" :message val
+                     :cause result))
+          (recur)))))))
+
+(defn- redis-pub
+  "Publish a message to the redis server. Asynchronous operation,
+  intended to be used in core.async go blocks."
+  [{:keys [::pconn] :as cfg} {:keys [topic message]}]
+  (let [message (t/encode message)
+        res     (a/chan 1)
+        pcomm   (.async ^StatefulRedisConnection pconn)]
+    (-> (.publish ^RedisAsyncCommands pcomm ^String topic ^bytes message)
+        (p/finally (fn [_ cause]
+                     (when (and cause (.isOpen ^StatefulConnection pconn))
+                       (a/offer! res cause))
                      (a/close! res))))
     res))
 
-(defn impl-redis-sub
-  [^RedisPubSubAsyncCommands rac topic]
-  (let [res (a/chan 1)]
-    (-> (.subscribe rac (into-array String [topic]))
-        (p/finally (fn [_ e]
-                     (when e (a/>!! res e))
-                     (a/close! res))))
-    res))
+(defn redis-sub
+  "Create redis subscription. Blocking operation, intended to be used
+  inside an agent."
+  [{:keys [::sconn] :as cfg} topic]
+  (let [topic (into-array String [topic])
+        scomm (.sync ^StatefulRedisPubSubConnection sconn)]
+    (.subscribe ^RedisPubSubCommands scomm topic)))
 
-(defn impl-redis-unsub
-  [rac topic]
-  (let [res (a/chan 1)]
-    (-> (.unsubscribe rac (into-array String [topic]))
-        (p/finally (fn [_ e]
-                     (when e (a/>!! res e))
-                     (a/close! res))))
-    res))
+(defn redis-unsub
+  "Removes redis subscription. Blocking operation, intended to be used
+  inside an agent."
+  [{:keys [::sconn] :as cfg} topic]
+  (let [topic (into-array String [topic])
+        scomm (.sync ^StatefulRedisPubSubConnection sconn)]
+    (.unsubscribe ^RedisPubSubCommands scomm topic)))

backend/src/app/notifications.clj

@@ -1,280 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.notifications
-  "A websocket based notifications mechanism."
-  (:require
-   [app.common.spec :as us]
-   [app.common.transit :as t]
-   [app.db :as db]
-   [app.metrics :as mtx]
-   [app.util.async :as aa]
-   [app.util.logging :as l]
-   [app.util.time :as dt]
-   [app.worker :as wrk]
-   [clojure.core.async :as a]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig]
-   [ring.adapter.jetty9 :as jetty]
-   [ring.middleware.cookies :refer [wrap-cookies]]
-   [ring.middleware.keyword-params :refer [wrap-keyword-params]]
-   [ring.middleware.params :refer [wrap-params]]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Http Handler
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(declare retrieve-file)
-(declare websocket)
-(declare handler)
-
-(s/def ::session map?)
-(s/def ::msgbus fn?)
-
-(defmethod ig/pre-init-spec ::handler [_]
-  (s/keys :req-un [::msgbus ::db/pool ::session ::mtx/metrics ::wrk/executor]))
-
-(defmethod ig/init-key ::handler
-  [_ {:keys [session metrics] :as cfg}]
-  (let [wrap-session    (:middleware session)
-
-        mtx-active-connections
-        (mtx/create
-         {:name "websocket_active_connections"
-          :registry (:registry metrics)
-          :type :gauge
-          :help "Active websocket connections."})
-
-        mtx-messages
-        (mtx/create
-         {:name "websocket_message_total"
-          :registry (:registry metrics)
-          :labels ["op"]
-          :type :counter
-          :help "Counter of processed messages."})
-
-        mtx-sessions
-        (mtx/create
-         {:name "websocket_session_timing"
-          :registry (:registry metrics)
-          :quantiles []
-          :help "Websocket session timing (seconds)."
-          :type :summary})
-
-        cfg (assoc cfg
-                   :mtx-active-connections mtx-active-connections
-                   :mtx-messages mtx-messages
-                   :mtx-sessions mtx-sessions
-                   )]
-    (-> #(handler cfg %)
-        (wrap-session)
-        (wrap-keyword-params)
-        (wrap-cookies)
-        (wrap-params))))
-
-(s/def ::file-id ::us/uuid)
-(s/def ::session-id ::us/uuid)
-
-(s/def ::websocket-handler-params
-  (s/keys :req-un [::file-id ::session-id]))
-
-(defn- handler
-  [{:keys [pool] :as cfg} {:keys [profile-id params] :as req}]
-  (let [params (us/conform ::websocket-handler-params params)
-        file   (retrieve-file pool (:file-id params))
-        cfg    (merge cfg params
-                      {:profile-id profile-id
-                       :team-id (:team-id file)})]
-    (cond
-      (not profile-id)
-      {:error {:code 403 :message "Authentication required"}}
-
-      (not file)
-      {:error {:code 404 :message "File does not exists"}}
-
-      :else
-      (websocket cfg))))
-
-(def ^:private
-  sql:retrieve-file
-  "select f.id as id,
-          p.team_id as team_id
-     from file as f
-     join project as p on (p.id = f.project_id)
-    where f.id = ?")
-
-(defn- retrieve-file
-  [conn id]
-  (db/exec-one! conn [sql:retrieve-file id]))
-
-
-;; --- WEBSOCKET INIT
-
-(declare handle-connect)
-
-(defn- ws-send
-  [conn data]
-  (try
-    (when (jetty/connected? conn)
-      (jetty/send! conn data)
-      true)
-    (catch java.lang.NullPointerException _e
-      false)))
-
-(defn websocket
-  [{:keys [file-id team-id msgbus executor] :as cfg}]
-  (let [rcv-ch       (a/chan 32)
-        out-ch       (a/chan 32)
-        mtx-aconn    (:mtx-active-connections cfg)
-        mtx-messages (:mtx-messages cfg)
-        mtx-sessions (:mtx-sessions cfg)
-        created-at   (dt/now)
-        ws-send      (mtx/wrap-counter ws-send mtx-messages ["send"])]
-
-    (letfn [(on-connect [conn]
-              (mtx-aconn :inc)
-              ;; A subscription channel should use a lossy buffer
-              ;; because we can't penalize normal clients when one
-              ;; slow client is connected to the room.
-              (let [sub-ch (a/chan (a/dropping-buffer 128))
-                    cfg    (assoc cfg
-                                  :conn conn
-                                  :rcv-ch rcv-ch
-                                  :out-ch out-ch
-                                  :sub-ch sub-ch)]
-
-                (l/trace :event "connect" :session (:session-id cfg))
-
-                ;; Forward all messages from out-ch to the websocket
-                ;; connection
-                (a/go-loop []
-                  (let [val (a/<! out-ch)]
-                    (when (some? val)
-                      (when (a/<! (aa/thread-call executor #(ws-send conn (t/encode-str val))))
-                        (recur)))))
-
-                (a/go
-                  ;; Subscribe to corresponding topics
-                  (a/<! (msgbus :sub {:topics [file-id team-id] :chan sub-ch}))
-                  (a/<! (handle-connect cfg))
-
-                  ;; when connection is closed
-                  (mtx-aconn :dec)
-                  (mtx-sessions :observe (/ (inst-ms (dt/diff created-at (dt/now))) 1000.0))
-
-                  ;; close subscription
-                  (a/close! sub-ch))))
-
-            (on-error [_conn _e]
-              (l/trace :event "error" :session (:session-id cfg))
-
-              (a/close! out-ch)
-              (a/close! rcv-ch))
-
-            (on-close [_conn _status _reason]
-              (l/trace :event "close" :session (:session-id cfg))
-
-              (a/close! out-ch)
-              (a/close! rcv-ch))
-
-            (on-message [_ws message]
-              (let [message (t/decode-str message)]
-                (when-not (a/offer! rcv-ch message)
-                  (l/warn :msg "drop messages"))))]
-
-      {:on-connect on-connect
-       :on-error on-error
-       :on-close on-close
-       :on-text (mtx/wrap-counter on-message mtx-messages ["recv"])
-       :on-bytes (constantly nil)})))
-
-;; --- CONNECTION INIT
-
-(declare send-presence)
-(declare handle-message)
-(declare start-loop!)
-
-(defn- handle-connect
-  [cfg]
-  (a/go
-    (a/<! (handle-message cfg {:type :connect}))
-    (a/<! (start-loop! cfg))
-    (a/<! (handle-message cfg {:type :disconnect}))))
-
-(defn- start-loop!
-  [{:keys [rcv-ch out-ch sub-ch session-id] :as cfg}]
-  (a/go-loop []
-    (let [timeout    (a/timeout 30000)
-          [val port] (a/alts! [rcv-ch sub-ch timeout])]
-      (cond
-        ;; Process message coming from connected client
-        (and (= port rcv-ch) (some? val))
-        (do
-          (a/<! (handle-message cfg val))
-          (recur))
-
-        ;; Process message coming from pubsub.
-        (and (= port sub-ch) (some? val))
-        (do
-          (when-not (= (:session-id val) session-id)
-            ;; If we receive a connect message of other user, we need
-            ;; to send an update presence to all participants.
-            (when (= :connect (:type val))
-              (a/<! (send-presence cfg :presence)))
-
-            ;; Then, just forward the message
-            (a/>! out-ch val))
-          (recur))
-
-        ;; When timeout channel is signaled, we need to send a ping
-        ;; message to the output channel. TODO: we need to make this
-        ;; more smart.
-        (= port timeout)
-        (do
-          (a/>! out-ch {:type :ping})
-          (recur))))))
-
-(defn send-presence
-  ([cfg] (send-presence cfg :presence))
-  ([{:keys [msgbus session-id profile-id file-id]} type]
-   (a/go
-     (a/<! (msgbus :pub {:topic file-id
-                         :message {:type type
-                                   :session-id session-id
-                                   :profile-id profile-id}})))))
-
-;; --- INCOMING MSG PROCESSING
-
-(defmulti handle-message
-  (fn [_ message] (:type message)))
-
-(defmethod handle-message :connect
-  [cfg _]
-  (send-presence cfg :connect))
-
-(defmethod handle-message :disconnect
-  [cfg _]
-  (send-presence cfg :disconnect))
-
-(defmethod handle-message :keepalive
-  [_ _]
-  (a/go :nothing))
-
-(defmethod handle-message :pointer-update
-  [{:keys [profile-id file-id session-id msgbus] :as cfg} message]
-  (let [message (assoc message
-                       :profile-id profile-id
-                       :session-id session-id)]
-    (msgbus :pub {:topic file-id
-                  :message message})))
-
-(defmethod handle-message :default
-  [_ws message]
-  (a/go
-    (l/log :level :warn
-           :msg "received unexpected message"
-           :message message)))
-

backend/src/app/rlimits.clj

@@ -1,45 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) UXBOX Labs SL
-
-(ns app.rlimits
-  "Resource usage limits (in other words: semaphores)."
-  (:require
-   [app.common.spec :as us]
-   [clojure.spec.alpha :as s]
-   [integrant.core :as ig])
-  (:import
-   java.util.concurrent.Semaphore))
-
-(s/def ::rlimit  #(instance? Semaphore %))
-(s/def ::rlimits (s/map-of ::us/keyword ::rlimit))
-
-(derive ::password ::instance)
-(derive ::image ::instance)
-(derive ::font ::instance)
-
-(defmethod ig/pre-init-spec ::instance [_]
-  (s/spec int?))
-
-(defmethod ig/init-key ::instance
-  [_ permits]
-  (Semaphore. (int permits)))
-
-(defn acquire!
-  [sem]
-  (.acquire ^Semaphore sem))
-
-(defn release!
-  [sem]
-  (.release ^Semaphore sem))
-
-(defmacro execute
-  [rlinst & body]
-  `(try
-     (acquire! ~rlinst)
-     ~@body
-     (finally
-       (release! ~rlinst))))
-

backend/src/app/rpc.clj

@@ -6,192 +6,300 @@
 
 (ns app.rpc
   (:require
-   [app.common.data :as d]
    [app.common.exceptions :as ex]
+   [app.common.logging :as l]
    [app.common.spec :as us]
    [app.db :as db]
    [app.loggers.audit :as audit]
    [app.metrics :as mtx]
-   [app.rlimits :as rlm]
-   [app.util.logging :as l]
+   [app.rpc.retry :as retry]
+   [app.rpc.rlimit :as rlimit]
+   [app.util.async :as async]
    [app.util.services :as sv]
+   [app.worker :as wrk]
    [clojure.spec.alpha :as s]
-   [cuerdas.core :as str]
-   [integrant.core :as ig]))
+   [integrant.core :as ig]
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [yetti.response :as yrs]))
 
 (defn- default-handler
   [_]
-  (ex/raise :type :not-found))
+  (p/rejected (ex/error :type :not-found)))
 
-(defn- run-hook
-  [hook-fn response]
-  (ex/ignoring (hook-fn))
+(defn- handle-response-transformation
+  [response request mdata]
+  (if-let [transform-fn (:transform-response mdata)]
+    (p/do (transform-fn request response))
+    (p/resolved response)))
+
+(defn- handle-before-comple-hook
+  [response mdata]
+  (when-let [hook-fn (:before-complete mdata)]
+    (ex/ignoring (hook-fn)))
   response)
 
 (defn- rpc-query-handler
-  [methods {:keys [profile-id] :as request}]
-  (let [type   (keyword (get-in request [:path-params :type]))
+  "Ring handler that dispatches query requests and convert between
+  internal async flow into ring async flow."
+  [methods {:keys [profile-id session-id params] :as request} respond raise]
+  (letfn [(handle-response [result]
+            (let [mdata (meta result)]
+              (-> (yrs/response 200 result)
+                  (handle-response-transformation request mdata))))]
 
-        data   (merge (:params request)
-                      (:body-params request)
-                      (:uploads request)
-                      {::request request})
+    (let [type   (keyword (:type params))
+          data   (into {::request request} params)
+          data   (if profile-id
+                   (assoc data :profile-id profile-id ::session-id session-id)
+                   (dissoc data :profile-id))
+          method (get methods type default-handler)]
 
-        data   (if profile-id
-                 (assoc data :profile-id profile-id)
-                 (dissoc data :profile-id))
-
-        result ((get methods type default-handler) data)
-        mdata  (meta result)]
-
-    (cond->> {:status 200 :body result}
-      (fn? (:transform-response mdata))
-      ((:transform-response mdata) request))))
+      (-> (method data)
+          (p/then handle-response)
+          (p/then respond)
+          (p/catch (fn [cause]
+                     (let [context {:profile-id profile-id}]
+                       (raise (ex/wrap-with-context cause context)))))))))
 
 (defn- rpc-mutation-handler
-  [methods {:keys [profile-id] :as request}]
-  (let [type   (keyword (get-in request [:path-params :type]))
-        data   (merge (:params request)
-                      (:body-params request)
-                      (:uploads request)
-                      {::request request})
+  "Ring handler that dispatches mutation requests and convert between
+  internal async flow into ring async flow."
+  [methods {:keys [profile-id session-id params] :as request} respond raise]
+  (letfn [(handle-response [result]
+            (let [mdata (meta result)]
+              (p/-> (yrs/response 200 result)
+                    (handle-response-transformation request mdata)
+                    (handle-before-comple-hook mdata))))]
 
-        data   (if profile-id
-                 (assoc data :profile-id profile-id)
-                 (dissoc data :profile-id))
+    (let [type   (keyword (:type params))
+          data   (into {::request request} params)
+          data   (if profile-id
+                   (assoc data :profile-id profile-id ::session-id session-id)
+                   (dissoc data :profile-id))
 
-        result ((get methods type default-handler) data)
-        mdata  (meta result)]
-    (cond->> {:status 200 :body result}
-      (fn? (:transform-response mdata))
-      ((:transform-response mdata) request)
+          method (get methods type default-handler)]
+      (-> (method data)
+          (p/then handle-response)
+          (p/then respond)
+          (p/catch (fn [cause]
+                     (let [context {:profile-id profile-id}]
+                       (raise (ex/wrap-with-context cause context)))))))))
 
-      (fn? (:before-complete mdata))
-      (run-hook (:before-complete mdata)))))
+(defn- rpc-command-handler
+  "Ring handler that dispatches cmd requests and convert between
+  internal async flow into ring async flow."
+  [methods {:keys [profile-id session-id params] :as request} respond raise]
+  (letfn [(handle-response [result]
+            (let [mdata (meta result)]
+              (p/-> (yrs/response 200 result)
+                    (handle-response-transformation request mdata)
+                    (handle-before-comple-hook mdata))))]
 
-(defn- wrap-with-metrics
-  [cfg f mdata]
-  (mtx/wrap-summary f (::mobj cfg) [(::sv/name mdata)]))
+    (let [cmd    (keyword (:command params))
+          data   (into {::request request} params)
+          data   (if profile-id
+                   (assoc data :profile-id profile-id ::session-id session-id)
+                   (dissoc data :profile-id))
 
-;; Wrap the rpc handler with a semaphore if it is specified in the
-;; metadata asocciated with the handler.
-(defn- wrap-with-rlimits
-  [cfg f mdata]
-  (if-let [key (:rlimit mdata)]
-    (let [rlinst (get-in cfg [:rlimits key])]
-      (when-not rlinst
-        (ex/raise :type :internal
-                  :code :rlimit-not-configured
-                  :hint (str/fmt "%s rlimit not configured" key)))
-      (l/trace :action "add rlimit"
-               :handler (::sv/name mdata))
-      (fn [cfg params]
-        (rlm/execute rlinst (f cfg params))))
+          method (get methods cmd default-handler)]
+      (-> (method data)
+          (p/then handle-response)
+          (p/then respond)
+          (p/catch (fn [cause]
+                     (let [context {:profile-id profile-id}]
+                       (raise (ex/wrap-with-context cause context)))))))))
+
+(defn- wrap-metrics
+  "Wrap service method with metrics measurement."
+  [{:keys [metrics ::metrics-id]} f mdata]
+  (let [labels (into-array String [(::sv/name mdata)])]
+    (fn [cfg params]
+      (let [start (System/nanoTime)]
+        (p/finally
+          (f cfg params)
+          (fn [_ _]
+            (mtx/run! metrics
+                      {:id metrics-id
+                       :val (/ (- (System/nanoTime) start) 1000000)
+                       :labels labels})))))))
+
+(defn- wrap-dispatch
+  "Wraps service method into async flow, with the ability to dispatching
+  it to a preconfigured executor service."
+  [{:keys [executors] :as cfg} f mdata]
+  (let [dname (::async/dispatch mdata :default)]
+    (if (= :none dname)
+      (with-meta
+        (fn [cfg params]
+          (p/do (f cfg params)))
+        mdata)
+
+      (let [executor (get executors dname)]
+        (when-not executor
+          (ex/raise :type :internal
+                    :code :executor-not-configured
+                    :hint (format "executor %s not configured" dname)))
+        (with-meta
+          (fn [cfg params]
+            (-> (px/submit! executor #(f cfg params))
+                (p/bind p/wrap)))
+          mdata)))))
+
+(defn- wrap-audit
+  [{:keys [audit] :as cfg} f mdata]
+  (if audit
+    (with-meta
+      (fn [cfg {:keys [::request] :as params}]
+        (p/finally (f cfg params)
+                   (fn [result _]
+                     (when result
+                       (let [resultm    (meta result)
+                             profile-id (or (::audit/profile-id resultm)
+                                            (:profile-id result)
+                                            (:profile-id params))
+                             props      (or (::audit/replace-props resultm)
+                                            (-> params
+                                                (merge (::audit/props resultm))
+                                                (dissoc :type)))]
+                         (audit :cmd :submit
+                                :type (or (::audit/type resultm)
+                                          (::type cfg))
+                                :name (or (::audit/name resultm)
+                                          (::sv/name mdata))
+                                :profile-id profile-id
+                                :ip-addr (some-> request audit/parse-client-ip)
+                                :props (dissoc props ::request)))))))
+      mdata)
     f))
 
-(defn- wrap-impl
-  [{:keys [audit] :as cfg} f mdata]
-  (let [f      (wrap-with-rlimits cfg f mdata)
-        f      (wrap-with-metrics cfg f mdata)
-        spec   (or (::sv/spec mdata) (s/spec any?))
-        auth?  (:auth mdata true)]
+(defn- wrap
+  [cfg f mdata]
+  (let [f     (as-> f $
+                (wrap-dispatch cfg $ mdata)
+                (rlimit/wrap-rlimit cfg $ mdata)
+                (retry/wrap-retry cfg $ mdata)
+                (wrap-audit cfg $ mdata)
+                (wrap-metrics cfg $ mdata)
+                )
 
-    (l/trace :action "register" :name (::sv/name mdata))
-    (fn [params]
+        spec  (or (::sv/spec mdata) (s/spec any?))
+        auth? (:auth mdata true)]
 
-      ;; Raise authentication error when rpc method requires auth but
-      ;; no profile-id is found in the request.
-      (when (and auth? (not (uuid? (:profile-id params))))
-        (ex/raise :type :authentication
-                  :code :authentication-required
-                  :hint "authentication required for this endpoint"))
+    (l/debug :hint "register method" :name (::sv/name mdata))
+    (with-meta
+      (fn [{:keys [::request] :as params}]
+        ;; Raise authentication error when rpc method requires auth but
+        ;; no profile-id is found in the request.
+        (p/do!
+         (if (and auth? (not (uuid? (:profile-id params))))
+           (ex/raise :type :authentication
+                     :code :authentication-required
+                     :hint "authentication required for this endpoint")
+           (let [params (us/conform spec (dissoc params ::request))]
+             (f cfg (assoc params ::request request))))))
 
-      (let [params' (dissoc params ::request)
-            params' (us/conform spec params')
-            result  (f cfg params')]
-
-        ;; When audit log is enabled (default false).
-        (when (fn? audit)
-          (let [resultm    (meta result)
-                request    (::request params)
-                profile-id (or (:profile-id params')
-                               (:profile-id result)
-                               (::audit/profile-id resultm))
-                props      (d/merge params (::audit/props resultm))]
-            (audit :cmd :submit
-                   :type (::type cfg)
-                   :name (or (::audit/name resultm)
-                             (::sv/name mdata))
-                   :profile-id profile-id
-                   :ip-addr (audit/parse-client-ip request)
-                   :props (audit/profile->props props))))
-
-        result))))
+      mdata)))
 
 (defn- process-method
   [cfg vfn]
   (let [mdata (meta vfn)]
     [(keyword (::sv/name mdata))
-     (wrap-impl cfg (deref vfn) mdata)]))
+     (wrap cfg vfn mdata)]))
 
 (defn- resolve-query-methods
   [cfg]
-  (let [mobj (mtx/create
-              {:name "rpc_query_timing"
-               :labels ["name"]
-               :registry (get-in cfg [:metrics :registry])
-               :type :histogram
-               :help "Timing of query services."})
-        cfg  (assoc cfg ::mobj mobj ::type "query")]
+  (let [cfg (assoc cfg ::type "query" ::metrics-id :rpc-query-timing)]
     (->> (sv/scan-ns 'app.rpc.queries.projects
                      'app.rpc.queries.files
                      'app.rpc.queries.teams
                      'app.rpc.queries.comments
                      'app.rpc.queries.profile
-                     'app.rpc.queries.recent-files
                      'app.rpc.queries.viewer
-                     'app.rpc.queries.fonts
-                     'app.rpc.queries.svg)
+                     'app.rpc.queries.fonts)
          (map (partial process-method cfg))
          (into {}))))
 
 (defn- resolve-mutation-methods
   [cfg]
-  (let [mobj (mtx/create
-              {:name "rpc_mutation_timing"
-               :labels ["name"]
-               :registry (get-in cfg [:metrics :registry])
-               :type :histogram
-               :help "Timing of mutation services."})
-        cfg  (assoc cfg ::mobj mobj ::type "mutation")]
-    (->> (sv/scan-ns 'app.rpc.mutations.demo
-                     'app.rpc.mutations.media
+  (let [cfg (assoc cfg ::type "mutation" ::metrics-id :rpc-mutation-timing)]
+    (->> (sv/scan-ns 'app.rpc.mutations.media
                      'app.rpc.mutations.profile
                      'app.rpc.mutations.files
                      'app.rpc.mutations.comments
                      'app.rpc.mutations.projects
-                     'app.rpc.mutations.viewer
                      'app.rpc.mutations.teams
                      'app.rpc.mutations.management
-                     'app.rpc.mutations.ldap
                      'app.rpc.mutations.fonts
+                     'app.rpc.mutations.share-link
                      'app.rpc.mutations.verify-token)
          (map (partial process-method cfg))
          (into {}))))
 
-(s/def ::storage some?)
-(s/def ::session map?)
-(s/def ::tokens fn?)
+(defn- resolve-command-methods
+  [cfg]
+  (let [cfg (assoc cfg ::type "command" ::metrics-id :rpc-command-timing)]
+    (->> (sv/scan-ns 'app.rpc.commands.binfile
+                     'app.rpc.commands.comments
+                     'app.rpc.commands.auth
+                     'app.rpc.commands.ldap
+                     'app.rpc.commands.demo
+                     'app.rpc.commands.files)
+         (map (partial process-method cfg))
+         (into {}))))
+
 (s/def ::audit (s/nilable fn?))
+(s/def ::executors (s/map-of keyword? ::wrk/executor))
+(s/def ::executors map?)
+(s/def ::http-client fn?)
+(s/def ::ldap (s/nilable map?))
+(s/def ::msgbus fn?)
+(s/def ::public-uri ::us/not-empty-string)
+(s/def ::session map?)
+(s/def ::storage some?)
+(s/def ::tokens fn?)
 
-(defmethod ig/pre-init-spec ::rpc [_]
-  (s/keys :req-un [::storage ::session ::tokens ::audit
-                   ::mtx/metrics ::rlm/rlimits ::db/pool]))
+(defmethod ig/pre-init-spec ::methods [_]
+  (s/keys :req-un [::storage
+                   ::session
+                   ::tokens
+                   ::audit
+                   ::executors
+                   ::public-uri
+                   ::msgbus
+                   ::http-client
+                   ::mtx/metrics
+                   ::db/pool
+                   ::ldap]))
 
-(defmethod ig/init-key ::rpc
+(defmethod ig/init-key ::methods
   [_ cfg]
-  (let [mq (resolve-query-methods cfg)
-        mm (resolve-mutation-methods cfg)]
-    {:methods {:query mq :mutation mm}
-     :query-handler #(rpc-query-handler mq %)
-     :mutation-handler #(rpc-mutation-handler mm %)}))
+  {:mutations (resolve-mutation-methods cfg)
+   :queries   (resolve-query-methods cfg)
+   :commands  (resolve-command-methods cfg)})
+
+(s/def ::mutations
+  (s/map-of keyword? fn?))
+
+(s/def ::queries
+  (s/map-of keyword? fn?))
+
+(s/def ::commands
+  (s/map-of keyword? fn?))
+
+(s/def ::methods
+  (s/keys :req-un [::mutations
+                   ::queries
+                   ::commands]))
+
+(defmethod ig/pre-init-spec ::routes [_]
+  (s/keys :req-un [::methods]))
+
+(defmethod ig/init-key ::routes
+  [_ {:keys [methods] :as cfg}]
+  [["/rpc"
+    ["/command/:command" {:handler (partial rpc-command-handler (:commands methods))}]
+    ["/query/:type" {:handler (partial rpc-query-handler (:queries methods))}]
+    ["/mutation/:type" {:handler (partial rpc-mutation-handler (:mutations methods))
+                        :allowed-methods #{:post}}]]])
+

backend/src/app/rpc/commands/auth.clj

@@ -0,0 +1,428 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.commands.auth
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.emails :as eml]
+   [app.loggers.audit :as audit]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.mutations.teams :as teams]
+   [app.rpc.queries.profile :as profile]
+   [app.rpc.rlimit :as rlimit]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [buddy.hashers :as hashers]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]))
+
+(s/def ::email ::us/email)
+(s/def ::fullname ::us/not-empty-string)
+(s/def ::lang ::us/string)
+(s/def ::path ::us/string)
+(s/def ::profile-id ::us/uuid)
+(s/def ::password ::us/not-empty-string)
+(s/def ::old-password ::us/not-empty-string)
+(s/def ::theme ::us/string)
+(s/def ::invitation-token ::us/not-empty-string)
+(s/def ::token ::us/not-empty-string)
+
+;; ---- HELPERS
+
+(defn derive-password
+  [password]
+  (hashers/derive password
+                  {:alg :argon2id
+                   :memory 16384
+                   :iterations 20
+                   :parallelism 2}))
+
+(defn verify-password
+  [attempt password]
+  (try
+    (hashers/verify attempt password)
+    (catch Exception _e
+      {:update false
+       :valid false})))
+
+(defn email-domain-in-whitelist?
+  "Returns true if email's domain is in the given whitelist or if
+  given whitelist is an empty string."
+  [domains email]
+  (if (or (empty? domains)
+          (nil? domains))
+    true
+    (let [[_ candidate] (-> (str/lower email)
+                            (str/split #"@" 2))]
+      (contains? domains candidate))))
+
+(def ^:private sql:profile-existence
+  "select exists (select * from profile
+                   where email = ?
+                     and deleted_at is null) as val")
+
+(defn check-profile-existence!
+  [conn {:keys [email] :as params}]
+  (let [email  (str/lower email)
+        result (db/exec-one! conn [sql:profile-existence email])]
+    (when (:val result)
+      (ex/raise :type :validation
+                :code :email-already-exists))
+    params))
+
+;; ---- COMMAND: login with password
+
+(defn login-with-password
+  [{:keys [pool session tokens] :as cfg} {:keys [email password] :as params}]
+
+  (when-not (contains? cf/flags :login)
+    (ex/raise :type :restriction
+              :code :login-disabled
+              :hint "login is disabled in this instance"))
+
+  (letfn [(check-password [profile password]
+            (when (= (:password profile) "!")
+              (ex/raise :type :validation
+                        :code :account-without-password
+                        :hint "the current account does not have password"))
+            (:valid (verify-password password (:password profile))))
+
+          (validate-profile [profile]
+            (when-not (:is-active profile)
+              (ex/raise :type :validation
+                        :code :wrong-credentials))
+            (when-not profile
+              (ex/raise :type :validation
+                        :code :wrong-credentials))
+            (when-not (check-password profile password)
+              (ex/raise :type :validation
+                        :code :wrong-credentials))
+            profile)]
+
+    (db/with-atomic [conn pool]
+      (let [profile    (->> (profile/retrieve-profile-data-by-email conn email)
+                            (validate-profile)
+                            (profile/strip-private-attrs)
+                            (profile/populate-additional-data conn)
+                            (profile/decode-profile-row))
+
+            invitation (when-let [token (:invitation-token params)]
+                         (tokens :verify {:token token :iss :team-invitation}))
+
+            ;; If invitation member-id does not matches the profile-id, we just proceed to ignore the
+            ;; invitation because invitations matches exactly; and user can't loging with other email and
+            ;; accept invitation with other email
+            response   (if (and (some? invitation) (= (:id profile) (:member-id invitation)))
+                         {:invitation-token (:invitation-token params)}
+                         profile)]
+
+        (with-meta response
+          {:transform-response ((:create session) (:id profile))
+           ::audit/props (audit/profile->props profile)
+           ::audit/profile-id (:id profile)})))))
+
+(s/def ::login-with-password
+  (s/keys :req-un [::email ::password]
+          :opt-un [::invitation-token]))
+
+(sv/defmethod ::login-with-password
+  "Performs authentication using penpot password."
+  {:auth false
+   ::rlimit/permits (cf/get :rlimit-password)
+   ::doc/added "1.15"}
+  [cfg params]
+  (login-with-password cfg params))
+
+;; ---- COMMAND: Logout
+
+(s/def ::logout
+  (s/keys :opt-un [::profile-id]))
+
+(sv/defmethod ::logout
+  "Clears the authentication cookie and logout the current session."
+  {:auth false
+   ::doc/added "1.15"}
+  [{:keys [session] :as cfg} _]
+  (with-meta {}
+    {:transform-response (:delete session)}))
+
+;; ---- COMMAND: Recover Profile
+
+(defn recover-profile
+  [{:keys [pool tokens] :as cfg} {:keys [token password]}]
+  (letfn [(validate-token [token]
+            (let [tdata (tokens :verify {:token token :iss :password-recovery})]
+              (:profile-id tdata)))
+
+          (update-password [conn profile-id]
+            (let [pwd (derive-password password)]
+              (db/update! conn :profile {:password pwd} {:id profile-id})))]
+
+    (db/with-atomic [conn pool]
+      (->> (validate-token token)
+           (update-password conn))
+      nil)))
+
+(s/def ::token ::us/not-empty-string)
+(s/def ::recover-profile
+  (s/keys :req-un [::token ::password]))
+
+(sv/defmethod ::recover-profile
+  {:auth false
+   ::rlimit/permits (cf/get :rlimit-password)
+   ::doc/added "1.15"}
+  [cfg params]
+  (recover-profile cfg params))
+
+;; ---- COMMAND: Prepare Register
+
+(defn prepare-register
+  [{:keys [pool tokens] :as cfg} params]
+  (when-not (contains? cf/flags :registration)
+    (if-not (contains? params :invitation-token)
+      (ex/raise :type :restriction
+                :code :registration-disabled)
+      (let [invitation (tokens :verify {:token (:invitation-token params) :iss :team-invitation})]
+        (when-not (= (:email params) (:member-email invitation))
+          (ex/raise :type :restriction
+                    :code :email-does-not-match-invitation
+                    :hint "email should match the invitation")))))
+
+  (when-let [domains (cf/get :registration-domain-whitelist)]
+    (when-not (email-domain-in-whitelist? domains (:email params))
+      (ex/raise :type :validation
+                :code :email-domain-is-not-allowed)))
+
+  ;; Don't allow proceed in preparing registration if the profile is
+  ;; already reported as spammer.
+  (when (eml/has-bounce-reports? pool (:email params))
+    (ex/raise :type :validation
+              :code :email-has-permanent-bounces
+              :hint "looks like the email has one or many bounces reported"))
+
+  (check-profile-existence! pool params)
+
+  (when (= (str/lower (:email params))
+           (str/lower (:password params)))
+    (ex/raise :type :validation
+              :code :email-as-password
+              :hint "you can't use your email as password"))
+
+  (let [params {:email (:email params)
+                :password (:password params)
+                :invitation-token (:invitation-token params)
+                :backend "penpot"
+                :iss :prepared-register
+                :exp (dt/in-future "48h")}
+
+        token  (tokens :generate params)]
+    (with-meta {:token token}
+      {::audit/profile-id uuid/zero})))
+
+(s/def ::prepare-register-profile
+  (s/keys :req-un [::email ::password]
+          :opt-un [::invitation-token]))
+
+(sv/defmethod ::prepare-register-profile
+  {:auth false
+   ::doc/added "1.15"}
+  [cfg params]
+  (prepare-register cfg params))
+
+;; ---- COMMAND: Register Profile
+
+(defn create-profile
+  "Create the profile entry on the database with limited input filling
+  all the other fields with defaults."
+  [conn params]
+  (let [id        (or (:id params) (uuid/next))
+
+        props     (-> (audit/extract-utm-params params)
+                      (merge (:props params))
+                      (db/tjson))
+
+        password  (if-let [password (:password params)]
+                    (derive-password password)
+                    "!")
+
+        locale    (:locale params)
+        locale    (when (and (string? locale) (not (str/blank? locale)))
+                    locale)
+
+        backend   (:backend params "penpot")
+        is-demo   (:is-demo params false)
+        is-muted  (:is-muted params false)
+        is-active (:is-active params false)
+        email     (str/lower (:email params))
+
+        params    {:id id
+                   :fullname (:fullname params)
+                   :email email
+                   :auth-backend backend
+                   :lang locale
+                   :password password
+                   :deleted-at (:deleted-at params)
+                   :props props
+                   :is-active is-active
+                   :is-muted is-muted
+                   :is-demo is-demo}]
+    (try
+      (-> (db/insert! conn :profile params)
+          (profile/decode-profile-row))
+      (catch org.postgresql.util.PSQLException e
+        (let [state (.getSQLState e)]
+          (if (not= state "23505")
+            (throw e)
+            (ex/raise :type :validation
+                      :code :email-already-exists
+                      :cause e)))))))
+
+(defn create-profile-relations
+  [conn profile]
+  (let [team (teams/create-team conn {:profile-id (:id profile)
+                                      :name "Default"
+                                      :is-default true})]
+    (-> profile
+        (profile/strip-private-attrs)
+        (assoc :default-team-id (:id team))
+        (assoc :default-project-id (:default-project-id team)))))
+
+(defn register-profile
+  [{:keys [conn tokens session] :as cfg} {:keys [token] :as params}]
+  (let [claims    (tokens :verify {:token token :iss :prepared-register})
+        params    (merge params claims)]
+    (check-profile-existence! conn params)
+    (let [is-active  (or (:is-active params)
+                         (contains? cf/flags :insecure-register))
+          profile    (->> (assoc params :is-active is-active)
+                          (create-profile conn)
+                          (create-profile-relations conn)
+                          (profile/decode-profile-row))
+          invitation (when-let [token (:invitation-token params)]
+                       (tokens :verify {:token token :iss :team-invitation}))]
+      (cond
+        ;; If invitation token comes in params, this is because the user comes from team-invitation process;
+        ;; in this case, regenerate token and send back to the user a new invitation token (and mark current
+        ;; session as logged). This happens only if the invitation email matches with the register email.
+        (and (some? invitation) (= (:email profile) (:member-email invitation)))
+        (let [claims (assoc invitation :member-id  (:id profile))
+              token  (tokens :generate claims)
+              resp   {:invitation-token token}]
+          (with-meta resp
+            {:transform-response ((:create session) (:id profile))
+             ::audit/replace-props (audit/profile->props profile)
+             ::audit/profile-id (:id profile)}))
+
+        ;; If auth backend is different from "penpot" means user is
+        ;; registering using third party auth mechanism; in this case
+        ;; we need to mark this session as logged.
+        (not= "penpot" (:auth-backend profile))
+        (with-meta (profile/strip-private-attrs profile)
+          {:transform-response ((:create session) (:id profile))
+           ::audit/replace-props (audit/profile->props profile)
+           ::audit/profile-id (:id profile)})
+
+        ;; If the `:enable-insecure-register` flag is set, we proceed
+        ;; to sign in the user directly, without email verification.
+        (true? is-active)
+        (with-meta (profile/strip-private-attrs profile)
+          {:transform-response ((:create session) (:id profile))
+           ::audit/replace-props (audit/profile->props profile)
+           ::audit/profile-id (:id profile)})
+
+        ;; In all other cases, send a verification email.
+        :else
+        (let [vtoken (tokens :generate
+                             {:iss :verify-email
+                              :exp (dt/in-future "48h")
+                              :profile-id (:id profile)
+                              :email (:email profile)})
+              ptoken (tokens :generate-predefined
+                             {:iss :profile-identity
+                              :profile-id (:id profile)})]
+          (eml/send! {::eml/conn conn
+                      ::eml/factory eml/register
+                      :public-uri (:public-uri cfg)
+                      :to (:email profile)
+                      :name (:fullname profile)
+                      :token vtoken
+                      :extra-data ptoken})
+
+          (with-meta profile
+            {::audit/replace-props (audit/profile->props profile)
+             ::audit/profile-id (:id profile)}))))))
+
+(s/def ::register-profile
+  (s/keys :req-un [::token ::fullname]))
+
+(sv/defmethod ::register-profile
+  {:auth false
+   ::rlimit/permits (cf/get :rlimit-password)
+   ::doc/added "1.15"}
+  [{:keys [pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (-> (assoc cfg :conn conn)
+        (register-profile params))))
+
+;; ---- COMMAND: Request Profile Recovery
+
+(defn request-profile-recovery
+  [{:keys [pool tokens] :as cfg} {:keys [email] :as params}]
+  (letfn [(create-recovery-token [{:keys [id] :as profile}]
+            (let [token (tokens :generate
+                                {:iss :password-recovery
+                                 :exp (dt/in-future "15m")
+                                 :profile-id id})]
+              (assoc profile :token token)))
+
+          (send-email-notification [conn profile]
+            (let [ptoken (tokens :generate-predefined
+                                 {:iss :profile-identity
+                                  :profile-id (:id profile)})]
+              (eml/send! {::eml/conn conn
+                          ::eml/factory eml/password-recovery
+                          :public-uri (:public-uri cfg)
+                          :to (:email profile)
+                          :token (:token profile)
+                          :name (:fullname profile)
+                          :extra-data ptoken})
+              nil))]
+
+    (db/with-atomic [conn pool]
+      (when-let [profile (profile/retrieve-profile-data-by-email conn email)]
+        (when-not (eml/allow-send-emails? conn profile)
+          (ex/raise :type :validation
+                    :code :profile-is-muted
+                    :hint "looks like the profile has reported repeatedly as spam or has permanent bounces."))
+
+        (when-not (:is-active profile)
+          (ex/raise :type :validation
+                    :code :profile-not-verified
+                    :hint "the user need to validate profile before recover password"))
+
+        (when (eml/has-bounce-reports? conn (:email profile))
+          (ex/raise :type :validation
+                    :code :email-has-permanent-bounces
+                    :hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
+
+        (->> profile
+             (create-recovery-token)
+             (send-email-notification conn))))))
+
+(s/def ::request-profile-recovery
+  (s/keys :req-un [::email]))
+
+(sv/defmethod ::request-profile-recovery
+  {:auth false
+   ::doc/added "1.15"}
+  [cfg params]
+  (request-profile-recovery cfg params))
+
+

backend/src/app/rpc/commands/binfile.clj

@@ -0,0 +1,868 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.commands.binfile
+  (:refer-clojure :exclude [assert])
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.pages.migrations :as pmg]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.media :as media]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.files :as files]
+   [app.rpc.queries.projects :as projects]
+   [app.storage :as sto]
+   [app.storage.tmp :as tmp]
+   [app.tasks.file-gc]
+   [app.util.blob :as blob]
+   [app.util.bytes :as bs]
+   [app.util.fressian :as fres]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [clojure.walk :as walk]
+   [cuerdas.core :as str]
+   [yetti.adapter :as yt])
+  (:import
+   java.io.DataInputStream
+   java.io.DataOutputStream
+   java.io.InputStream
+   java.io.OutputStream
+   java.lang.AutoCloseable))
+
+(set! *warn-on-reflection* true)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; DEFAULTS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Threshold in MiB when we pass from using
+;; in-memory byte-array's to use temporal files.
+(def temp-file-threshold
+  (* 1024 1024 2))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; LOW LEVEL STREAM IO API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def ^:const buffer-size (:xnio/buffer-size yt/defaults))
+(def ^:const penpot-magic-number 800099563638710213)
+(def ^:const max-object-size (* 1024 1024 100)) ; Only allow 100MiB max file size.
+
+(def ^:dynamic *position* nil)
+
+(defn get-mark
+  [id]
+  (case id
+    :header  1
+    :stream  2
+    :uuid    3
+    :label   4
+    :obj     5
+    (ex/raise :type :validation
+              :code :invalid-mark-id
+              :hint (format "invalid mark id %s" id))))
+
+(defmacro assert
+  [expr hint]
+  `(when-not ~expr
+     (ex/raise :type :validation
+               :code :unexpected-condition
+               :hint ~hint)))
+
+(defmacro assert-mark
+  [v type]
+  `(let [expected# (get-mark ~type)
+         val#      (long ~v)]
+     (when (not= val# expected#)
+       (ex/raise :type :validation
+                 :code :unexpected-mark
+                 :hint (format "received mark %s, expected %s" val# expected#)))))
+
+(defmacro assert-label
+  [expr label]
+  `(let [v# ~expr]
+     (when (not= v# ~label)
+       (ex/raise :type :assertion
+                 :code :unexpected-label
+                 :hint (format "received label %s, expected %s" v# ~label)))))
+
+;; --- PRIMITIVE IO
+
+(defn write-byte!
+  [^DataOutputStream output data]
+  (l/trace :fn "write-byte!" :data data :position @*position* ::l/async false)
+  (.writeByte output (byte data))
+  (swap! *position* inc))
+
+(defn read-byte!
+  [^DataInputStream input]
+  (let [v (.readByte input)]
+    (l/trace :fn "read-byte!" :val v :position @*position* ::l/async false)
+    (swap! *position* inc)
+    v))
+
+(defn write-long!
+  [^DataOutputStream output data]
+  (l/trace :fn "write-long!" :data data :position @*position* ::l/async false)
+  (.writeLong output (long data))
+  (swap! *position* + 8))
+
+
+(defn read-long!
+  [^DataInputStream input]
+  (let [v (.readLong input)]
+    (l/trace :fn "read-long!" :val v :position @*position* ::l/async false)
+    (swap! *position* + 8)
+    v))
+
+(defn write-bytes!
+  [^DataOutputStream output ^bytes data]
+  (let [size (alength data)]
+    (l/trace :fn "write-bytes!" :size size :position @*position* ::l/async false)
+    (.write output data 0 size)
+    (swap! *position* + size)))
+
+(defn read-bytes!
+  [^InputStream input ^bytes buff]
+  (let [size   (alength buff)
+        readed (.readNBytes input buff 0 size)]
+    (l/trace :fn "read-bytes!" :expected (alength buff) :readed readed :position @*position* ::l/async false)
+    (swap! *position* + readed)
+    readed))
+
+;; --- COMPOSITE IO
+
+(defn write-uuid!
+  [^DataOutputStream output id]
+  (l/trace :fn "write-uuid!" :position @*position* :WRITTEN? (.size output) ::l/async false)
+
+  (doto output
+    (write-byte! (get-mark :uuid))
+    (write-long! (uuid/get-word-high id))
+    (write-long! (uuid/get-word-low id))))
+
+(defn read-uuid!
+  [^DataInputStream input]
+  (l/trace :fn "read-uuid!" :position @*position* ::l/async false)
+  (let [m (read-byte! input)]
+    (assert-mark m :uuid)
+    (let [a (read-long! input)
+          b (read-long! input)]
+      (uuid/custom a b))))
+
+(defn write-obj!
+  [^DataOutputStream output data]
+  (l/trace :fn "write-obj!" :position @*position* ::l/async false)
+  (let [^bytes data (fres/encode data)]
+    (doto output
+      (write-byte! (get-mark :obj))
+      (write-long! (alength data))
+      (write-bytes! data))))
+
+(defn read-obj!
+  [^DataInputStream input]
+  (l/trace :fn "read-obj!" :position @*position* ::l/async false)
+  (let [m (read-byte! input)]
+    (assert-mark m :obj)
+    (let [size (read-long! input)]
+      (assert (pos? size) "incorrect header size found on reading header")
+      (let [buff (byte-array size)]
+        (read-bytes! input buff)
+        (fres/decode buff)))))
+
+(defn write-label!
+  [^DataOutputStream output label]
+  (l/trace :fn "write-label!" :label label :position @*position* ::l/async false)
+  (doto output
+    (write-byte! (get-mark :label))
+    (write-obj! label)))
+
+(defn read-label!
+  [^DataInputStream input]
+  (l/trace :fn "read-label!" :position @*position* ::l/async false)
+  (let [m (read-byte! input)]
+    (assert-mark m :label)
+    (read-obj! input)))
+
+(defn write-header!
+  [^OutputStream output version]
+  (l/trace :fn "write-header!"
+           :version version
+           :position @*position*
+           ::l/async false)
+  (let [vers   (-> version name (subs 1) parse-long)
+        output (bs/data-output-stream output)]
+    (doto output
+      (write-byte! (get-mark :header))
+      (write-long! penpot-magic-number)
+      (write-long! vers))))
+
+(defn read-header!
+  [^InputStream input]
+  (l/trace :fn "read-header!" :position @*position* ::l/async false)
+  (let [input (bs/data-input-stream input)
+        mark  (read-byte! input)
+        mnum  (read-long! input)
+        vers  (read-long! input)]
+
+    (when (or (not= mark (get-mark :header))
+              (not= mnum penpot-magic-number))
+      (ex/raise :type :validation
+                :code :invalid-penpot-file
+                :hint "invalid penpot file"))
+
+    (keyword (str "v" vers))))
+
+(defn copy-stream!
+  [^OutputStream output ^InputStream input ^long size]
+  (let [written (bs/copy! input output :size size)]
+    (l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/async false)
+    (swap! *position* + written)
+    written))
+
+(defn write-stream!
+  [^DataOutputStream output stream size]
+  (l/trace :fn "write-stream!" :position @*position* ::l/async false :size size)
+  (doto output
+    (write-byte! (get-mark :stream))
+    (write-long! size))
+
+  (copy-stream! output stream size))
+
+(defn read-stream!
+  [^DataInputStream input]
+  (l/trace :fn "read-stream!" :position @*position* ::l/async false)
+  (let [m (read-byte! input)
+        s (read-long! input)
+        p (tmp/tempfile :prefix "penpot.binfile.")]
+    (assert-mark m :stream)
+
+    (when (> s max-object-size)
+      (ex/raise :type :validation
+                :code :max-file-size-reached
+                :hint (str/ffmt "unable to import storage object with size % bytes" s)))
+
+    (if (> s temp-file-threshold)
+      (with-open [^OutputStream output (io/output-stream p)]
+        (let [readed (bs/copy! input output :offset 0 :size s)]
+          (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/async false)
+          (swap! *position* + readed)
+          [s p]))
+      [s (bs/read-as-bytes input :size s)])))
+
+(defmacro assert-read-label!
+  [input expected-label]
+  `(let [readed# (read-label! ~input)
+         expected# ~expected-label]
+     (when (not= readed# expected#)
+       (ex/raise :type :validation
+                 :code :unexpected-label
+                 :hint (format "unxpected label found: %s, expected: %s" readed# expected#)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; --- HELPERS
+
+(defn- retrieve-file
+  [pool file-id]
+  (->> (db/query pool :file {:id file-id})
+       (map files/decode-row)
+       (first)))
+
+(def ^:private sql:file-media-objects
+  "SELECT * FROM file_media_object WHERE id = ANY(?)")
+
+(defn- retrieve-file-media
+  [pool {:keys [data id] :as file}]
+  (with-open [^AutoCloseable conn (db/open pool)]
+    (let [ids (app.tasks.file-gc/collect-used-media data)
+          ids (db/create-array conn "uuid" ids)]
+
+      ;; We assoc the file-id again to the file-media-object row
+      ;; because there are cases that used objects refer to other
+      ;; files and we need to ensure in the exportation process that
+      ;; all ids matches
+      (->> (db/exec! conn [sql:file-media-objects ids])
+           (mapv #(assoc % :file-id id))))))
+
+(def ^:private storage-object-id-xf
+  (comp
+   (mapcat (juxt :media-id :thumbnail-id))
+   (filter uuid?)))
+
+(def ^:private sql:file-libraries
+  "WITH RECURSIVE libs AS (
+     SELECT fl.id, fl.deleted_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+      WHERE flr.file_id = ANY(?)
+    UNION
+     SELECT fl.id, fl.deleted_at
+       FROM file AS fl
+       JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
+       JOIN libs AS l ON (flr.file_id = l.id)
+   )
+   SELECT DISTINCT l.id
+     FROM libs AS l
+    WHERE l.deleted_at IS NULL OR l.deleted_at > now();")
+
+(defn- retrieve-libraries
+  [pool ids]
+  (with-open [^AutoCloseable conn (db/open pool)]
+    (let [ids (db/create-array conn "uuid" ids)]
+      (map :id (db/exec! pool [sql:file-libraries ids])))))
+
+(def ^:private sql:file-library-rels
+  "SELECT * FROM file_library_rel
+    WHERE file_id = ANY(?)")
+
+(defn- retrieve-library-relations
+  [pool ids]
+  (with-open [^AutoCloseable conn (db/open pool)]
+    (db/exec! conn [sql:file-library-rels (db/create-array conn "uuid" ids)])))
+
+
+(defn- create-or-update-file
+  [conn params]
+  (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) "
+                 "VALUES (?, ?, ?, ?, ?, ?, ?, ?) "
+                 "ON CONFLICT (id) DO UPDATE SET data=?")]
+    (db/exec-one! conn [sql
+                        (:id params)
+                        (:project-id params)
+                        (:name params)
+                        (:revn params)
+                        (:is-shared params)
+                        (:data params)
+                        (:created-at params)
+                        (:modified-at params)
+                        (:data params)])))
+
+;; --- GENERAL PURPOSE DYNAMIC VARS
+
+(def ^:dynamic *state*)
+(def ^:dynamic *options*)
+
+;; --- EXPORT WRITTER
+
+(defn- embed-file-assets
+  [data conn file-id]
+  (letfn [(walk-map-form [form state]
+            (cond
+              (uuid? (:fill-color-ref-file form))
+              (do
+                (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)])
+                (assoc form :fill-color-ref-file file-id))
+
+              (uuid? (:stroke-color-ref-file form))
+              (do
+                (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)])
+                (assoc form :stroke-color-ref-file file-id))
+
+              (uuid? (:typography-ref-file form))
+              (do
+                (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)])
+                (assoc form :typography-ref-file file-id))
+
+              (uuid? (:component-file form))
+              (do
+                (vswap! state conj [(:component-file form) :components (:component-id form)])
+                (assoc form :component-file file-id))
+
+              :else
+              form))
+
+          (process-group-of-assets [data [lib-id items]]
+            ;; NOTE: there are a posibility that shape refers to a not
+            ;; existing file because the file was removed. In this
+            ;; case we just ignore the asset.
+            (if-let [lib (retrieve-file conn lib-id)]
+              (reduce (partial process-asset lib) data items)
+              data))
+
+          (process-asset [lib data [bucket asset-id]]
+            (let [asset (get-in lib [:data bucket asset-id])
+                  ;; Add a special case for colors that need to have
+                  ;; correctly set the :file-id prop (pending of the
+                  ;; refactor that will remove it).
+                  asset (cond-> asset
+                          (= bucket :colors) (assoc :file-id file-id))]
+              (update data bucket assoc asset-id asset)))]
+
+    (let [assets (volatile! [])]
+      (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data)
+      (->> (deref assets)
+           (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id))))
+           (d/group-by first rest)
+           (reduce (partial process-group-of-assets) data)))))
+
+(defmulti write-export ::version)
+(defmulti write-section ::section)
+
+(s/def ::output bs/output-stream?)
+(s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1))
+(s/def ::include-libraries? (s/nilable ::us/boolean))
+(s/def ::embed-assets? (s/nilable ::us/boolean))
+
+(s/def ::write-export-options
+  (s/keys :req-un [::db/pool ::sto/storage]
+          :req    [::output ::file-ids]
+          :opt    [::include-libraries? ::embed-assets?]))
+
+(defn write-export!
+  "Do the exportation of a speficied file in custom penpot binary
+  format. There are some options available for customize the output:
+
+  `::include-libraries?`: additionaly to the specified file, all the
+  linked libraries also will be included (including transitive
+  dependencies).
+
+  `::embed-assets?`: instead of including the libraryes, embedd in the
+  same file library all assets used from external libraries."
+  [{:keys [::include-libraries? ::embed-assets?] :as options}]
+  (us/assert! ::write-export-options options)
+  (us/verify!
+   :expr (not (and include-libraries? embed-assets?))
+   :hint "the `include-libraries?` and `embed-assets?` are mutally excluding options")
+  (write-export options))
+
+(defmethod write-export :default
+  [{:keys [::output] :as options}]
+  (write-header! output :v1)
+  (with-open [output (bs/zstd-output-stream output :level 12)]
+    (with-open [output (bs/data-output-stream output)]
+      (binding [*state* (volatile! {})]
+        (run! (fn [section]
+                (l/debug :hint "write section" :section section ::l/async false)
+                (write-label! output section)
+                (let [options (-> options
+                                  (assoc ::output output)
+                                  (assoc ::section section))]
+                  (binding [*options* options]
+                    (write-section options))))
+
+              [:v1/metadata :v1/files :v1/rels :v1/sobjects])))))
+
+(defmethod write-section :v1/metadata
+  [{:keys [pool ::output ::file-ids ::include-libraries?]}]
+  (let [libs  (when include-libraries?
+                (retrieve-libraries pool file-ids))
+        files (into file-ids libs)]
+    (write-obj! output {:version cf/version :files files})
+    (vswap! *state* assoc :files files)))
+
+(defmethod write-section :v1/files
+  [{:keys [pool ::output ::embed-assets?]}]
+
+  ;; Initialize SIDS with empty vector
+  (vswap! *state* assoc :sids [])
+
+  (doseq [file-id (-> *state* deref :files)]
+    (let [file  (cond-> (retrieve-file pool file-id)
+                  embed-assets?
+                  (update :data embed-file-assets pool file-id))
+
+          media (retrieve-file-media pool file)]
+
+      (l/debug :hint "write penpot file"
+               :id file-id
+               :media (count media)
+               ::l/async false)
+
+      (doto output
+        (write-obj! file)
+        (write-obj! media))
+
+      (vswap! *state* update :sids into storage-object-id-xf media))))
+
+(defmethod write-section :v1/rels
+  [{:keys [pool  ::output ::include-libraries?]}]
+  (let [rels  (when include-libraries?
+                (retrieve-library-relations pool (-> *state* deref :files)))]
+    (l/debug :hint "found rels" :total (count rels) ::l/async false)
+    (write-obj! output rels)))
+
+(defmethod write-section :v1/sobjects
+  [{:keys [storage ::output]}]
+  (let [sids    (-> *state* deref :sids)
+        storage (media/configure-assets-storage storage)]
+    (l/debug :hint "found sobjects"
+             :items (count sids)
+             ::l/async false)
+
+    ;; Write all collected storage objects
+    (write-obj! output sids)
+
+    (doseq [id sids]
+      (let [{:keys [size] :as obj} @(sto/get-object storage id)]
+        (l/debug :hint "write sobject" :id id ::l/async false)
+        (doto output
+          (write-uuid! id)
+          (write-obj! (meta obj)))
+
+        (with-open [^InputStream stream @(sto/get-object-data storage obj)]
+          (let [written (write-stream! output stream size)]
+            (when (not= written size)
+              (ex/raise :type :validation
+                        :code :mismatch-readed-size
+                        :hint (str/ffmt "found unexpected object size; size=% written=%" size written)))))))))
+
+;; --- EXPORT READER
+
+(declare lookup-index)
+(declare update-index)
+(declare relink-media)
+(declare relink-shapes)
+
+(defmulti read-import ::version)
+(defmulti read-section ::section)
+
+(s/def ::project-id ::us/uuid)
+(s/def ::input bs/input-stream?)
+(s/def ::overwrite? (s/nilable ::us/boolean))
+(s/def ::migrate? (s/nilable ::us/boolean))
+(s/def ::ignore-index-errors? (s/nilable ::us/boolean))
+
+(s/def ::read-import-options
+  (s/keys :req-un [::db/pool ::sto/storage]
+          :req    [::project-id ::input]
+          :opt    [::overwrite? ::migrate? ::ignore-index-errors?]))
+
+(defn read-import!
+  "Do the importation of the specified resource in penpot custom binary
+  format. There are some options for customize the importation
+  behavior:
+
+  `::overwrite?`: if true, instead of creating new files and remaping id references,
+  it reuses all ids and updates existing objects; defaults to `false`.
+
+  `::migrate?`: if true, applies the migration before persisting the
+  file data; defaults to `false`.
+
+  `::ignore-index-errors?`: if true, do not fail on index lookup errors, can
+  happen with broken files; defaults to: `false`.
+  "
+
+  [{:keys [::input ::timestamp] :or {timestamp (dt/now)} :as options}]
+  (us/verify! ::read-import-options options)
+  (let [version (read-header! input)]
+    (read-import (assoc options ::version version ::timestamp timestamp))))
+
+(defmethod read-import :v1
+  [{:keys [pool ::input] :as options}]
+  (with-open [input (bs/zstd-input-stream input)]
+    (with-open [input (bs/data-input-stream input)]
+      (db/with-atomic [conn pool]
+        (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED;"])
+        (binding [*state* (volatile! {:media [] :index {}})]
+          (run! (fn [section]
+                  (l/debug :hint "reading section" :section section ::l/async false)
+                  (assert-read-label! input section)
+                  (let [options (-> options
+                                    (assoc ::section section)
+                                    (assoc ::input input)
+                                    (assoc :conn conn))]
+                    (binding [*options* options]
+                      (read-section options))))
+                [:v1/metadata :v1/files :v1/rels :v1/sobjects])
+
+          ;; Knowing that the ids of the created files are in
+          ;; index, just lookup them and return it as a set
+          (let [files (-> *state* deref :files)]
+            (into #{} (keep #(get-in @*state* [:index %])) files)))))))
+
+(defmethod read-section :v1/metadata
+  [{:keys [::input]}]
+  (let [{:keys [version files]} (read-obj! input)]
+    (l/debug :hint "metadata readed" :version (:full version) :files files ::l/async false)
+    (vswap! *state* update :index update-index files)
+    (vswap! *state* assoc :version version :files files)))
+
+(defmethod read-section :v1/files
+  [{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
+  (doseq [expected-file-id (-> *state* deref :files)]
+    (let [file    (read-obj! input)
+          media'  (read-obj! input)
+          file-id (:id file)]
+
+      (when (not= file-id expected-file-id)
+        (ex/raise :type :validation
+                  :code :inconsistent-penpot-file
+                  :hint "the penpot file seems corrupt, found unexpected uuid (file-id)"))
+
+      ;; Update index using with media
+      (l/debug :hint "update index with media" ::l/async false)
+      (vswap! *state* update :index update-index (map :id media'))
+
+      ;; Store file media for later insertion
+      (l/debug :hint "update media references" ::l/async false)
+      (vswap! *state* update :media into (map #(update % :id lookup-index)) media')
+
+      (l/debug :hint "procesing file" :file-id file-id ::l/async false)
+
+      (let [file-id' (lookup-index file-id)
+            data     (-> (:data file)
+                         (assoc :id file-id')
+                         (cond-> migrate? (pmg/migrate-data))
+                         (update :pages-index relink-shapes)
+                         (update :components relink-shapes)
+                         (update :media relink-media))
+
+            params  {:id file-id'
+                     :project-id project-id
+                     :name (str "Imported: " (:name file))
+                     :revn (:revn file)
+                     :is-shared (:is-shared file)
+                     :data (blob/encode data)
+                     :created-at timestamp
+                     :modified-at timestamp}]
+
+        (l/debug :hint "create file" :id file-id' ::l/async false)
+
+        (if overwrite?
+          (create-or-update-file conn params)
+          (db/insert! conn :file params))
+
+        (when overwrite?
+          (db/delete! conn :file-thumbnail {:file-id file-id'}))))))
+
+(defmethod read-section :v1/rels
+  [{:keys [conn ::input ::timestamp]}]
+  (let [rels (read-obj! input)]
+    ;; Insert all file relations
+    (doseq [rel rels]
+      (let [rel (-> rel
+                    (assoc :synced-at timestamp)
+                    (update :file-id lookup-index)
+                    (update :library-file-id lookup-index))]
+        (l/debug :hint "create file library link"
+                 :file-id (:file-id rel)
+                 :lib-id (:library-file-id rel)
+                 ::l/async false)
+        (db/insert! conn :file-library-rel rel)))))
+
+(defmethod read-section :v1/sobjects
+  [{:keys [storage conn ::input ::overwrite?]}]
+  (let [storage (media/configure-assets-storage storage)
+        ids     (read-obj! input)]
+
+    (doseq [expected-storage-id ids]
+      (let [id    (read-uuid! input)
+            mdata (read-obj! input)]
+
+        (when (not= id expected-storage-id)
+          (ex/raise :type :validation
+                    :code :inconsistent-penpot-file
+                    :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)"))
+
+        (l/debug :hint "readed storage object" :id id ::l/async false)
+
+        (let [[size resource] (read-stream! input)
+              hash            (sto/calculate-hash resource)
+              content         (-> (sto/content resource size)
+                                  (sto/wrap-with-hash hash))
+              params          (-> mdata
+                                  (assoc ::sto/deduplicate? true)
+                                  (assoc ::sto/content content)
+                                  (assoc ::sto/touched-at (dt/now))
+                                  (assoc :bucket "file-media-object"))
+
+              sobject         @(sto/put-object! storage params)]
+
+          (l/debug :hint "persisted storage object" :id id :new-id (:id sobject) ::l/async false)
+          (vswap! *state* update :index assoc id (:id sobject)))))
+
+    (doseq [item (:media @*state*)]
+      (l/debug :hint "inserting file media object"
+               :id (:id item)
+               :file-id (:file-id item)
+               ::l/async false)
+
+      (let [file-id (lookup-index (:file-id item))]
+        (if (= file-id (:file-id item))
+          (l/warn :hint "ignoring file media object" :file-id (:file-id item) ::l/async false)
+          (db/insert! conn :file-media-object
+                      (-> item
+                          (assoc :file-id file-id)
+                          (d/update-when :media-id lookup-index)
+                          (d/update-when :thumbnail-id lookup-index))
+                      {:on-conflict-do-nothing overwrite?}))))))
+
+(defn- lookup-index
+  [id]
+  (let [val (get-in @*state* [:index id])]
+    (l/trace :fn "lookup-index" :id id :val val ::l/async false)
+    (when (and (not (::ignore-index-errors? *options*)) (not val))
+      (ex/raise :type :validation
+                :code :incomplete-index
+                :hint "looks like index has missing data"))
+    (or val id)))
+
+(defn- update-index
+  [index coll]
+  (loop [items (seq coll)
+         index index]
+    (if-let [id (first items)]
+      (let [new-id (if (::overwrite? *options*) id (uuid/next))]
+        (l/trace :fn "update-index" :id id :new-id new-id ::l/async false)
+        (recur (rest items)
+               (assoc index id new-id)))
+      index)))
+
+(defn- relink-shapes
+  "A function responsible to analyze all file data and
+  replace the old :component-file reference with the new
+  ones, using the provided file-index."
+  [data]
+  (letfn [(process-map-form [form]
+            (cond-> form
+              ;; Relink image shapes
+              (and (map? (:metadata form))
+                   (= :image (:type form)))
+              (update-in [:metadata :id] lookup-index)
+
+              ;; Relink paths with fill image
+              (and (map? (:fill-image form))
+                   (= :path (:type form)))
+              (update-in [:fill-image :id] lookup-index)
+
+              ;; This covers old shapes and the new :fills.
+              (uuid? (:fill-color-ref-file form))
+              (update :fill-color-ref-file lookup-index)
+
+              ;; This covers the old shapes and the new :strokes
+              (uuid? (:storage-color-ref-file form))
+              (update :stroke-color-ref-file lookup-index)
+
+              ;; This covers all text shapes that have typography referenced
+              (uuid? (:typography-ref-file form))
+              (update :typography-ref-file lookup-index)
+
+              ;; This covers the shadows and grids (they have directly
+              ;; the :file-id prop)
+              (uuid? (:file-id form))
+              (update :file-id lookup-index)))]
+
+    (walk/postwalk (fn [form]
+                     (if (map? form)
+                       (try
+                         (process-map-form form)
+                         (catch Throwable cause
+                           (l/warn :hint "failed form" :form (pr-str form) ::l/async false)
+                           (throw cause)))
+                       form))
+                   data)))
+
+(defn- relink-media
+  "A function responsible of process the :media attr of file data and
+  remap the old ids with the new ones."
+  [media]
+  (reduce-kv (fn [res k v]
+               (let [id (lookup-index k)]
+                 (if (uuid? id)
+                   (-> res
+                       (assoc id (assoc v :id id))
+                       (dissoc k))
+                   res)))
+             media
+             media))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; HIGH LEVEL API
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn export!
+  [cfg]
+  (let [path (tmp/tempfile :prefix "penpot.export.")
+        id   (uuid/next)
+        ts   (dt/now)
+        cs   (volatile! nil)]
+    (try
+      (l/info :hint "start exportation" :export-id id)
+      (with-open [output (io/output-stream path)]
+        (binding [*position* (atom 0)]
+          (write-export! (assoc cfg ::output output))
+          path))
+
+      (catch Throwable cause
+        (vreset! cs cause)
+        (throw cause))
+
+      (finally
+        (l/info :hint "exportation finished" :export-id id
+                :elapsed (str (inst-ms (dt/diff ts (dt/now))) "ms")
+                :cause @cs)))))
+
+(defn import!
+  [{:keys [::input] :as cfg}]
+  (let [id (uuid/next)
+        ts (dt/now)
+        cs (volatile! nil)]
+    (try
+      (l/info :hint "start importation" :import-id id)
+      (binding [*position* (atom 0)]
+        (with-open [input (io/input-stream input)]
+          (read-import! (assoc cfg ::input input))))
+
+      (catch Throwable cause
+        (vreset! cs cause)
+        (throw cause))
+
+      (finally
+        (l/info :hint "importation finished" :import-id id
+                :elapsed (str (inst-ms (dt/diff ts (dt/now))) "ms")
+                :error? (some? @cs)
+                :cause @cs)))))
+
+;; --- Command: export-binfile
+
+(s/def ::file-id ::us/uuid)
+(s/def ::profile-id ::us/uuid)
+(s/def ::include-libraries? ::us/boolean)
+(s/def ::embed-assets? ::us/boolean)
+
+(s/def ::export-binfile
+  (s/keys :req-un [::profile-id ::file-id ::include-libraries? ::embed-assets?]))
+
+(sv/defmethod ::export-binfile
+  "Export a penpot file in a binary format."
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id include-libraries? embed-assets?] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-read-permissions! conn profile-id file-id)
+    (let [path (export! (assoc cfg
+                               ::file-ids [file-id]
+                               ::embed-assets? embed-assets?
+                               ::include-libraries? include-libraries?))]
+      (with-meta {}
+        {:transform-response (fn [_ response]
+                               (assoc response
+                                      :body (io/input-stream path)
+                                      :headers {"content-type" "application/octet-stream"}))}))))
+
+(s/def ::file ::media/upload)
+(s/def ::import-binfile
+  (s/keys :req-un [::profile-id ::project-id ::file]))
+
+(sv/defmethod ::import-binfile
+  "Import a penpot file in a binary format."
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id project-id file] :as params}]
+  (db/with-atomic [conn pool]
+    (projects/check-read-permissions! conn profile-id project-id)
+    (import! (assoc cfg
+                    ::input (:path file)
+                    ::project-id project-id
+                    ::ignore-index-errors? true))))

backend/src/app/rpc/commands/comments.clj

@@ -0,0 +1,532 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.commands.comments
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.geom.point :as gpt]
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.files :as files]
+   [app.rpc.queries.teams :as teams]
+   [app.rpc.retry :as retry]
+   [app.util.blob :as blob]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [clojure.spec.alpha :as s]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; QUERY COMMANDS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn decode-row
+  [{:keys [participants position] :as row}]
+  (cond-> row
+    (db/pgpoint? position) (assoc :position (db/decode-pgpoint position))
+    (db/pgobject? participants) (assoc :participants (db/decode-transit-pgobject participants))))
+
+;; --- COMMAND: Get Comment Threads
+
+(declare retrieve-comment-threads)
+
+(s/def ::team-id ::us/uuid)
+(s/def ::file-id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+
+(s/def ::get-comment-threads
+  (s/and (s/keys :req-un [::profile-id]
+                 :opt-un [::file-id ::share-id ::team-id])
+         #(or (:file-id %) (:team-id %))))
+
+(sv/defmethod ::get-comment-threads
+  [{:keys [pool] :as cfg} params]
+  (with-open [conn (db/open pool)]
+    (retrieve-comment-threads conn params)))
+
+(def sql:comment-threads
+  "select distinct on (ct.id)
+          ct.*,
+          f.name as file_name,
+          f.project_id as project_id,
+          first_value(c.content) over w as content,
+          (select count(1)
+             from comment as c
+            where c.thread_id = ct.id) as count_comments,
+          (select count(1)
+             from comment as c
+            where c.thread_id = ct.id
+              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
+     from comment_thread as ct
+    inner join comment as c on (c.thread_id = ct.id)
+    inner join file as f on (f.id = ct.file_id)
+     left join comment_thread_status as cts
+            on (cts.thread_id = ct.id and
+                cts.profile_id = ?)
+    where ct.file_id = ?
+   window w as (partition by c.thread_id order by c.created_at asc)")
+
+(defn retrieve-comment-threads
+  [conn {:keys [profile-id file-id share-id]}]
+  (files/check-comment-permissions! conn profile-id file-id share-id)
+  (->> (db/exec! conn [sql:comment-threads profile-id file-id])
+       (into [] (map decode-row))))
+
+;; --- COMMAND: Get Unread Comment Threads
+
+(declare retrieve-unread-comment-threads)
+
+(s/def ::team-id ::us/uuid)
+(s/def ::get-unread-comment-threads
+  (s/keys :req-un [::profile-id ::team-id]))
+
+(sv/defmethod ::get-unread-comment-threads
+  [{:keys [pool] :as cfg} {:keys [profile-id team-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (teams/check-read-permissions! conn profile-id team-id)
+    (retrieve-unread-comment-threads conn params)))
+
+(def sql:comment-threads-by-team
+  "select distinct on (ct.id)
+          ct.*,
+          f.name as file_name,
+          f.project_id as project_id,
+          first_value(c.content) over w as content,
+          (select count(1)
+             from comment as c
+            where c.thread_id = ct.id) as count_comments,
+          (select count(1)
+             from comment as c
+            where c.thread_id = ct.id
+              and c.created_at >= coalesce(cts.modified_at, ct.created_at)) as count_unread_comments
+     from comment_thread as ct
+    inner join comment as c on (c.thread_id = ct.id)
+    inner join file as f on (f.id = ct.file_id)
+    inner join project as p on (p.id = f.project_id)
+     left join comment_thread_status as cts
+            on (cts.thread_id = ct.id and
+                cts.profile_id = ?)
+    where p.team_id = ?
+   window w as (partition by c.thread_id order by c.created_at asc)")
+
+(def sql:unread-comment-threads-by-team
+  (str "with threads as (" sql:comment-threads-by-team ")"
+       "select * from threads where count_unread_comments > 0"))
+
+(defn retrieve-unread-comment-threads
+  [conn {:keys [profile-id team-id]}]
+  (->> (db/exec! conn [sql:unread-comment-threads-by-team profile-id team-id])
+       (into [] (map decode-row))))
+
+
+;; --- COMMAND: Get Single Comment Thread
+
+(s/def ::id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+(s/def ::get-comment-thread
+  (s/keys :req-un [::profile-id ::file-id ::id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::get-comment-thread
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id id share-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (let [sql (str "with threads as (" sql:comment-threads ")"
+                   "select * from threads where id = ?")]
+      (-> (db/exec-one! conn [sql profile-id file-id id])
+          (decode-row)))))
+
+(defn get-comment-thread
+  [conn {:keys [profile-id file-id id] :as params}]
+  (let [sql (str "with threads as (" sql:comment-threads ")"
+                 "select * from threads where id = ?")]
+    (-> (db/exec-one! conn [sql profile-id file-id id])
+        (decode-row))))
+
+;; --- COMMAND: Retrieve Comments
+
+(declare get-comments)
+
+(s/def ::file-id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+(s/def ::thread-id ::us/uuid)
+(s/def ::get-comments
+  (s/keys :req-un [::profile-id ::thread-id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::get-comments
+  [{:keys [pool] :as cfg} {:keys [profile-id thread-id share-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (let [thread (db/get-by-id conn :comment-thread thread-id)]
+      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id))
+    (get-comments conn thread-id)))
+
+(def sql:comments
+  "select c.* from comment as c
+    where c.thread_id = ?
+    order by c.created_at asc")
+
+(defn get-comments
+  [conn thread-id]
+  (->> (db/query conn :comment
+                 {:thread-id thread-id}
+                 {:order-by [[:created-at :asc]]})
+       (into [] (map decode-row))))
+
+;; --- COMMAND: Get file comments users
+
+(declare get-file-comments-users)
+
+(s/def ::file-id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+
+(s/def ::get-profiles-for-file-comments
+  (s/keys :req-un [::profile-id ::file-id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::get-profiles-for-file-comments
+  "Retrieves a list of profiles with limited set of properties of all
+  participants on comment threads of the file."
+  {::doc/added "1.15"
+   ::doc/changes ["1.15" "Imported from queries and renamed."]}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id]}]
+  (with-open [conn (db/open pool)]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (get-file-comments-users conn file-id profile-id)))
+
+;; All the profiles that had comment the file, plus the current
+;; profile.
+
+(def sql:file-comment-users
+  "WITH available_profiles AS (
+     SELECT DISTINCT owner_id AS id
+       FROM comment
+      WHERE thread_id IN (SELECT id FROM comment_thread WHERE file_id=?)
+  )
+  SELECT p.id,
+         p.email,
+         p.fullname AS name,
+         p.fullname AS fullname,
+         p.photo_id,
+         p.is_active
+    FROM profile AS p
+   WHERE p.id IN (SELECT id FROM available_profiles) OR p.id=?")
+
+(defn get-file-comments-users
+  [conn file-id profile-id]
+  (db/exec! conn [sql:file-comment-users file-id profile-id]))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; MUTATION COMMANDS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; --- COMMAND: Create Comment Thread
+
+(declare upsert-comment-thread-status!)
+(declare create-comment-thread)
+(declare retrieve-page-name)
+
+(s/def ::page-id ::us/uuid)
+(s/def ::file-id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+(s/def ::profile-id ::us/uuid)
+(s/def ::position ::gpt/point)
+(s/def ::content ::us/string)
+(s/def ::frame-id ::us/uuid)
+
+(s/def ::create-comment-thread
+  (s/keys :req-un [::profile-id ::file-id ::position ::content ::page-id ::frame-id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::create-comment-thread
+  {::retry/max-retries 3
+   ::retry/matches retry/conflict-db-insert?
+   ::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}]
+  (db/with-atomic [conn pool]
+    (files/check-comment-permissions! conn profile-id file-id share-id)
+    (create-comment-thread conn params)))
+
+(defn- retrieve-next-seqn
+  [conn file-id]
+  (let [sql "select (f.comment_thread_seqn + 1) as next_seqn from file as f where f.id = ?"
+        res (db/exec-one! conn [sql file-id])]
+    (:next-seqn res)))
+
+(defn create-comment-thread
+  [conn {:keys [profile-id file-id page-id position content frame-id] :as params}]
+  (let [seqn    (retrieve-next-seqn conn file-id)
+        now     (dt/now)
+        pname   (retrieve-page-name conn params)
+        thread  (db/insert! conn :comment-thread
+                           {:file-id file-id
+                            :owner-id profile-id
+                            :participants (db/tjson #{profile-id})
+                            :page-name pname
+                            :page-id page-id
+                            :created-at now
+                            :modified-at now
+                            :seqn seqn
+                            :position (db/pgpoint position)
+                            :frame-id frame-id})]
+
+
+    ;; Create a comment entry
+    (db/insert! conn :comment
+                {:thread-id (:id thread)
+                 :owner-id profile-id
+                 :created-at now
+                 :modified-at now
+                 :content content})
+
+    ;; Make the current thread as read.
+    (upsert-comment-thread-status! conn profile-id (:id thread))
+
+    ;; Optimistic update of current seq number on file.
+    (db/update! conn :file
+                {:comment-thread-seqn seqn}
+                {:id file-id})
+
+    (select-keys thread [:id :file-id :page-id])))
+
+(defn- retrieve-page-name
+  [conn {:keys [file-id page-id]}]
+  (let [{:keys [data]} (db/get-by-id conn :file file-id)
+        data           (blob/decode data)]
+    (get-in data [:pages-index page-id :name])))
+
+
+;; --- COMMAND: Update Comment Thread Status
+
+(s/def ::id ::us/uuid)
+(s/def ::share-id (s/nilable ::us/uuid))
+
+(s/def ::update-comment-thread-status
+  (s/keys :req-un [::profile-id ::id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::update-comment-thread-status
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id share-id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [cthr (db/get-by-id conn :comment-thread id {:for-update true})]
+      (when-not cthr
+        (ex/raise :type :not-found))
+
+      (files/check-comment-permissions! conn profile-id (:file-id cthr) share-id)
+      (upsert-comment-thread-status! conn profile-id (:id cthr)))))
+
+(def sql:upsert-comment-thread-status
+  "insert into comment_thread_status (thread_id, profile_id)
+   values (?, ?)
+       on conflict (thread_id, profile_id)
+       do update set modified_at = clock_timestamp()
+   returning modified_at;")
+
+(defn upsert-comment-thread-status!
+  [conn profile-id thread-id]
+  (db/exec-one! conn [sql:upsert-comment-thread-status thread-id profile-id]))
+
+
+;; --- COMMAND: Update Comment Thread
+
+(s/def ::is-resolved ::us/boolean)
+(s/def ::update-comment-thread
+  (s/keys :req-un [::profile-id ::id ::is-resolved]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::update-comment-thread
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id is-resolved share-id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [thread (db/get-by-id conn :comment-thread id {:for-update true})]
+      (when-not thread
+        (ex/raise :type :not-found))
+
+      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+
+      (db/update! conn :comment-thread
+                  {:is-resolved is-resolved}
+                  {:id id})
+      nil)))
+
+
+;; --- COMMAND: Add Comment
+
+(declare create-comment)
+
+(s/def ::create-comment
+  (s/keys :req-un [::profile-id ::thread-id ::content]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::create-comment
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (create-comment conn params)))
+
+(defn create-comment
+  [conn {:keys [profile-id thread-id content share-id] :as params}]
+  (let [thread (-> (db/get-by-id conn :comment-thread thread-id {:for-update true})
+                   (decode-row))
+        pname  (retrieve-page-name conn thread)]
+
+    ;; Standard Checks
+    (when-not thread (ex/raise :type :not-found))
+
+    ;; Permission Checks
+    (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+
+    ;; Update the page-name cachedattribute on comment thread table.
+    (when (not= pname (:page-name thread))
+      (db/update! conn :comment-thread
+                  {:page-name pname}
+                  {:id thread-id}))
+
+    ;; NOTE: is important that all timestamptz related fields are
+    ;; created or updated on the database level for avoid clock
+    ;; inconsistencies (some user sees something read that is not
+    ;; read, etc...)
+    (let [ppants  (:participants thread #{})
+          comment (db/insert! conn :comment
+                              {:thread-id thread-id
+                               :owner-id profile-id
+                               :content content})]
+
+      ;; NOTE: this is done in SQL instead of using db/update!
+      ;; helper because currently the helper does not allow pass raw
+      ;; function call parameters to the underlying prepared
+      ;; statement; in a future when we fix/improve it, this can be
+      ;; changed to use the helper.
+
+      ;; Update thread modified-at attribute and assoc the current
+      ;; profile to the participant set.
+      (let [ppants (conj ppants profile-id)
+            sql    "update comment_thread
+                         set modified_at = clock_timestamp(),
+                             participants = ?
+                       where id = ?"]
+        (db/exec-one! conn [sql (db/tjson ppants) thread-id]))
+
+      ;; Update the current profile status in relation to the
+      ;; current thread.
+      (upsert-comment-thread-status! conn profile-id thread-id)
+
+      ;; Return the created comment object.
+      comment)))
+
+;; --- COMMAND: Update Comment
+
+(declare update-comment)
+
+(s/def ::update-comment
+  (s/keys :req-un [::profile-id ::id ::content]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::update-comment
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} params]
+  (db/with-atomic [conn pool]
+    (update-comment conn params)))
+
+(defn update-comment
+  [conn {:keys [profile-id id content share-id] :as params}]
+  (let [comment (db/get-by-id conn :comment id {:for-update true})
+        _       (when-not comment (ex/raise :type :not-found))
+        thread  (db/get-by-id conn :comment-thread (:thread-id comment) {:for-update true})
+        _       (when-not thread (ex/raise :type :not-found))
+        pname   (retrieve-page-name conn thread)]
+
+    (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+
+    ;; Don't allow edit comments to not owners
+    (when-not (= (:owner-id thread) profile-id)
+      (ex/raise :type :validation
+                :code :not-allowed))
+
+    (db/update! conn :comment
+                {:content content
+                 :modified-at (dt/now)}
+                {:id (:id comment)})
+
+    (db/update! conn :comment-thread
+                {:modified-at (dt/now)
+                 :page-name pname}
+                {:id (:id thread)})
+    nil))
+
+
+;; --- COMMAND: Delete Comment Thread
+
+(s/def ::delete-comment-thread
+  (s/keys :req-un [::profile-id ::id]))
+
+(sv/defmethod ::delete-comment-thread
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [thread (db/get-by-id conn :comment-thread id {:for-update true})]
+      (when-not (= (:owner-id thread) profile-id)
+        (ex/raise :type :validation
+                  :code :not-allowed))
+      (db/delete! conn :comment-thread {:id id})
+      nil)))
+
+
+;; --- COMMAND: Delete comment
+
+(s/def ::delete-comment
+  (s/keys :req-un [::profile-id ::id]))
+
+(sv/defmethod ::delete-comment
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [comment (db/get-by-id conn :comment id {:for-update true})]
+      (when-not (= (:owner-id comment) profile-id)
+        (ex/raise :type :validation
+                  :code :not-allowed))
+
+      (db/delete! conn :comment {:id id}))))
+
+;; --- COMMAND: Update comment thread position
+
+(s/def ::update-comment-thread-position
+  (s/keys :req-un [::profile-id ::id ::position ::frame-id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::update-comment-thread-position
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id position frame-id share-id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [thread (db/get-by-id conn :comment-thread id {:for-update true})]
+      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+      (db/update! conn :comment-thread
+                  {:modified-at (dt/now)
+                   :position (db/pgpoint position)
+                   :frame-id frame-id}
+                  {:id (:id thread)})
+      nil)))
+
+;; --- COMMAND: Update comment frame
+
+(s/def ::update-comment-thread-frame
+  (s/keys :req-un [::profile-id ::id ::frame-id]
+          :opt-un [::share-id]))
+
+(sv/defmethod ::update-comment-thread-frame
+  {::doc/added "1.15"}
+  [{:keys [pool] :as cfg} {:keys [profile-id id frame-id share-id] :as params}]
+  (db/with-atomic [conn pool]
+    (let [thread (db/get-by-id conn :comment-thread id {:for-update true})]
+      (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)
+      (db/update! conn :comment-thread
+                  {:modified-at (dt/now)
+                   :frame-id frame-id}
+                  {:id (:id thread)})
+      nil)))
+

backend/src/app/rpc/commands/demo.clj

@@ -4,16 +4,16 @@
 ;;
 ;; Copyright (c) UXBOX Labs SL
 
-(ns app.rpc.mutations.demo
+(ns app.rpc.commands.demo
   "A demo specific mutations."
   (:require
    [app.common.exceptions :as ex]
    [app.common.uuid :as uuid]
-   [app.config :as cfg]
+   [app.config :as cf]
    [app.db :as db]
    [app.loggers.audit :as audit]
-   [app.rpc.mutations.profile :as profile]
-   [app.setup.initial-data :as sid]
+   [app.rpc.commands.auth :as cmd.auth]
+   [app.rpc.doc :as-alias doc]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [buddy.core.codecs :as bc]
@@ -22,7 +22,13 @@
 
 (s/def ::create-demo-profile any?)
 
-(sv/defmethod ::create-demo-profile {:auth false}
+(sv/defmethod ::create-demo-profile
+  "A command that is responsible of creating a demo purpose
+  profile. It only works if the `demo-users` flag is inabled in the
+  configuration."
+  {:auth false
+   ::doc/added "1.15"
+   ::doc/changes ["1.15" "This methos is migrated from mutations to commands."]}
   [{:keys [pool] :as cfg} _]
   (let [id       (uuid/next)
         sem      (System/currentTimeMillis)
@@ -34,20 +40,20 @@
         params   {:id id
                   :email email
                   :fullname fullname
-                  :is-demo true
-                  :deleted-at (dt/in-future cfg/deletion-delay)
+                  :is-active true
+                  :deleted-at (dt/in-future cf/deletion-delay)
                   :password password
-                  :props {:onboarding-viewed true}}]
+                  :props {}
+                  }]
 
-    (when-not (cfg/get :allow-demo-users)
+    (when-not (contains? cf/flags :demo-users)
       (ex/raise :type :validation
                 :code :demo-users-not-allowed
                 :hint "Demo users are disabled by config."))
 
     (db/with-atomic [conn pool]
-      (->> (#'profile/create-profile conn params)
-           (#'profile/create-profile-relations conn)
-           (sid/load-initial-project! conn))
+      (->> (cmd.auth/create-profile conn params)
+           (cmd.auth/create-profile-relations conn))
 
       (with-meta {:email email
                   :password password}

backend/src/app/rpc/commands/files.clj

@@ -0,0 +1,50 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.commands.files
+  (:require
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.files :as files]
+   [app.util.services :as sv]
+   [clojure.spec.alpha :as s]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; QUERY COMMANDS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; --- Query: File Libraries used by a File
+
+(declare retrieve-has-file-libraries)
+
+(s/def ::file-id ::us/uuid)
+(s/def ::profile-id ::us/uuid)
+
+(s/def ::has-file-libraries
+  (s/keys :req-un [::profile-id ::file-id]))
+
+(sv/defmethod ::has-file-libraries
+  "Checks if the file has libraries. Returns a boolean"
+  {::doc/added "1.15.1"}
+  [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}]
+  (with-open [conn (db/open pool)]
+    (files/check-read-permissions! pool profile-id file-id)
+    (retrieve-has-file-libraries conn params)))
+
+(def ^:private sql:has-file-libraries
+  "SELECT COUNT(*) > 0 AS has_libraries
+     FROM file_library_rel AS flr
+     JOIN file AS fl ON (flr.library_file_id = fl.id)
+    WHERE flr.file_id = ?::uuid
+      AND (fl.deleted_at IS NULL OR
+           fl.deleted_at > now())")
+
+(defn- retrieve-has-file-libraries
+  [conn {:keys [file-id]}]
+  (let [row (db/exec-one! conn [sql:has-file-libraries file-id])]
+    (:has-libraries row)))
+

backend/src/app/rpc/commands/ldap.clj

@@ -0,0 +1,80 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.commands.ldap
+  (:require
+   [app.auth.ldap :as ldap]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.db :as db]
+   [app.loggers.audit :as-alias audit]
+   [app.rpc.commands.auth :as cmd.auth]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.profile :as profile]
+   [app.util.services :as sv]
+   [clojure.spec.alpha :as s]))
+
+;; --- COMMAND: login-with-ldap
+
+(declare login-or-register)
+
+(s/def ::email ::us/email)
+(s/def ::password ::us/string)
+(s/def ::invitation-token ::us/string)
+
+(s/def ::login-with-ldap
+  (s/keys :req-un [::email ::password]
+          :opt-un [::invitation-token]))
+
+(sv/defmethod ::login-with-ldap
+  "Performs the authentication using LDAP backend. Only works if LDAP
+  is properly configured and enabled with `login-with-ldap` flag."
+  {:auth false
+   ::doc/added "1.15"}
+  [{:keys [session tokens ldap] :as cfg} params]
+  (when-not ldap
+    (ex/raise :type :restriction
+              :code :ldap-not-initialized
+              :hide "ldap auth provider is not initialized"))
+
+  (let [info (ldap/authenticate ldap params)]
+    (when-not info
+      (ex/raise :type :validation
+                :code :wrong-credentials))
+
+    (let [profile (login-or-register cfg info)]
+      (if-let [token (:invitation-token params)]
+        ;; If invitation token comes in params, this is because the
+        ;; user comes from team-invitation process; in this case,
+        ;; regenerate token and send back to the user a new invitation
+        ;; token (and mark current session as logged).
+        (let [claims (tokens :verify {:token token :iss :team-invitation})
+              claims (assoc claims
+                            :member-id  (:id profile)
+                            :member-email (:email profile))
+              token  (tokens :generate claims)]
+          (with-meta {:invitation-token token}
+            {:transform-response ((:create session) (:id profile))
+             ::audit/props (:props profile)
+             ::audit/profile-id (:id profile)}))
+
+        (with-meta profile
+          {:transform-response ((:create session) (:id profile))
+           ::audit/props (:props profile)
+           ::audit/profile-id (:id profile)})))))
+
+(defn- login-or-register
+  [{:keys [pool] :as cfg} info]
+  (db/with-atomic [conn pool]
+    (or (some->> (:email info)
+                 (profile/retrieve-profile-data-by-email conn)
+                 (profile/populate-additional-data conn)
+                 (profile/decode-profile-row))
+        (->> (assoc info :is-active true :is-demo false)
+             (cmd.auth/create-profile conn)
+             (cmd.auth/create-profile-relations conn)
+             (profile/strip-private-attrs)))))
+

backend/src/app/rpc/doc.clj

@@ -0,0 +1,77 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.rpc.doc
+  "API autogenerated documentation."
+  (:require
+   [app.common.data :as d]
+   [app.config :as cf]
+   [app.rpc :as-alias rpc]
+   [app.util.services :as sv]
+   [app.util.template :as tmpl]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]
+   [pretty-spec.core :as ps]
+   [yetti.response :as yrs]))
+
+(defn- get-spec-str
+  [k]
+  (with-out-str
+    (ps/pprint (s/form k)
+               {:ns-aliases {"clojure.spec.alpha" "s"
+                             "clojure.core.specs.alpha" "score"
+                             "clojure.core" nil}})))
+
+(defn- prepare-context
+  [methods]
+  (letfn [(gen-doc [type [name f]]
+            (let [mdata (meta f)]
+              {:type (d/name type)
+               :name (d/name name)
+               :module (-> (:ns mdata) (str/split ".") last)
+               :auth (:auth mdata true)
+               :docs (::sv/docstring mdata)
+               :deprecated (::deprecated mdata)
+               :added (::added mdata)
+               :changes (some->> (::changes mdata) (partition-all 2) (map vec))
+               :spec (get-spec-str (::sv/spec mdata))}))]
+
+    {:version (:main cf/version)
+     :command-methods
+     (->> (:commands methods)
+          (map (partial gen-doc :command))
+          (sort-by (juxt :module :name)))
+
+     :query-methods
+     (->> (:queries methods)
+          (map (partial gen-doc :query))
+          (sort-by (juxt :module :name)))
+     :mutation-methods
+     (->> (:mutations methods)
+          (map (partial gen-doc :query))
+          (sort-by (juxt :module :name)))}))
+
+(defn- handler
+  [methods]
+  (if (contains? cf/flags :backend-api-doc)
+    (let [context (prepare-context methods)]
+      (fn [_ respond _]
+        (respond (yrs/response 200 (-> (io/resource "api-doc.tmpl")
+                                       (tmpl/render context))))))
+    (fn [_ respond _]
+      (respond (yrs/response 404)))))
+
+
+(defmethod ig/pre-init-spec ::routes [_]
+  (s/keys :req-un [::rpc/methods]))
+
+(defmethod ig/init-key ::routes
+  [_ {:keys [methods] :as cfg}]
+  ["/_doc" {:handler (handler methods)
+            :allowed-methods #{:get}}])
+