📎 Update changelog and increase version.

When term is empty on frontend, frontend just does not sends it
to backend, leving it as missing field. This commit makes the
seatch-term as optional.

.circleci/config.yml

@@ -0,0 +1,67 @@
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: penpotapp/devenv:latest
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+      - image: circleci/postgres:13.1-ram
+        environment:
+          POSTGRES_USER: penpot_test
+          POSTGRES_PASSWORD: penpot_test
+          POSTGRES_DB: penpot_test
+
+      - image: circleci/redis:6.0.8
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx1g
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+         keys:
+           - v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}
+           # fallback to using the latest cache if no exact match is found
+           - v1-dependencies-
+
+      # run lint
+      - run:
+          working_directory: "./backend"
+          name: backend lint
+          command: "clj-kondo --lint src/"
+
+      # run test
+      - run:
+          working_directory: "./backend"
+          name: backend test
+          command: "clojure -M:dev:tests"
+          environment:
+            PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
+            PENPOT_TEST_DATABASE_USERNAME: penpot_test
+            PENPOT_TEST_DATABASE_PASSWORD: penpot_test
+            PENPOT_TEST_REDIS_URI: "redis://localhost/1"
+
+      - run:
+          working_directory: "./frontend"
+          name: frontend tests
+          command: |
+            yarn install
+            npx shadow-cljs compile tests
+          environment:
+            JAVA_HOME: /usr/lib/jvm/openjdk16
+            PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
+
+      - save_cache:
+         paths:
+           - ~/.m2
+         key: v1-dependencies-{{ checksum "backend/deps.edn" }}-{{ checksum "frontend/deps.edn"}}
+

.clj-kondo/config.edn

@@ -1,6 +1,12 @@
 {:lint-as {potok.core/reify clojure.core/reify
            promesa.core/let clojure.core/let
+           rumext.alpha/defc clojure.core/defn
+           app.common.data/export clojure.core/def
            app.db/with-atomic clojure.core/with-open}
+
+ :hooks
+ {:analyze-call {app.common.data/export hooks.export/export}}
+
  :output
  {:exclude-files ["data_readers.clj"]}
 
@@ -21,9 +27,6 @@
    }
 
   :unresolved-symbol
-  {:exclude ['(app.services.mutations/defmutation)
-             '(app.services.queries/defquery)
-             '(app.util.dispatcher/defservice)
-             '(mount.core/defstate)
+  {:exclude ['(app.util.services/defmethod)
              ]}}}
 

.clj-kondo/hooks/export.clj

@@ -0,0 +1,11 @@
+(ns hooks.export
+  (:require [clj-kondo.hooks-api :as api]))
+
+(defn export
+  [{:keys [:node]}]
+  (let [[_ sname] (:children node)
+        result  (api/list-node
+                 [(api/token-node (symbol "def"))
+                  (api/token-node (symbol (name (:value sname))))
+                  sname])]
+    {:node result}))

.github/ISSUE_TEMPLATE/bug_report.md

@@ -38,13 +38,13 @@ If applicable, add screenshots to help explain your problem.
 -   Version (e.g. 22)
 
 **Environment (please complete the following information):**
-Specify if using demo instance or self-hosted instance.
+Specify if using SAAS (https://design.penpot.app) or self-hosted instance.
 
 If self-hosted instance, add OS and runtime information to help explain your problem.
 
 -   OS Version: (e.g. Ubuntu 16.04)
 
-Also provide Docker commands or docker-compose file if possible.
+Also provide Docker commands or docker-compose file if possible and if proceed.x
 
 -   Docker / Docker-compose Version: (e.g. Docker version 18.03.0-ce, build 0520e24)
 -   Image (e.g. alpine)

.gitignore

@@ -14,21 +14,28 @@ figwheel_server.log
 node_modules
 /backend/target/
 /backend/resources/public/media
+/backend/resources/public/assets
+/backend/assets/
 /backend/dist/
 /backend/logs/
 /backend/-
+/telemetry/
 /frontend/npm-debug.log
 /frontend/target/
 /frontend/dist/
 /frontend/out/
 /frontend/.shadow-cljs
 /frontend/resources/public/*
+/frontend/resources/fonts/experiments
 /exporter/target
 /exporter/.shadow-cljs
-/docker/images/bundle
+/docker/images/bundle*
+/common/.shadow-cljs
+/common/target
 /.clj-kondo/.cache
 /bundle*
 /media
 /deploy
 /web
 /_dump
+/vendor/svgclean/bundle*.js

.gitpod.yml

@@ -0,0 +1,105 @@
+image:
+  file: docker/gitpod/Dockerfile
+
+ports:
+  # nginx
+  - port: 3449
+    onOpen: open-preview
+
+  # frontend nREPL
+  - port: 3447
+    onOpen: ignore
+    visibility: private
+
+  # frontend shadow server
+  - port: 3448
+    onOpen: ignore
+    visibility: private
+
+  # backend
+  - port: 6060
+    onOpen: ignore
+
+  # exporter shadow server
+  - port: 9630
+    onOpen: ignore
+    visibility: private
+
+  # exporter http server
+  - port: 6061
+    onOpen: ignore
+
+  # mailhog web interface
+  - port: 8025
+    onOpen: ignore
+
+  # mailhog postfix
+  - port: 1025
+    onOpen: ignore
+
+  # postgres
+  - port: 5432
+    onOpen: ignore
+
+  # redis
+  - port: 6379
+    onOpen: ignore
+
+  # openldap
+  - port: 389
+    onOpen: ignore
+
+tasks:
+  # https://github.com/gitpod-io/gitpod/issues/666#issuecomment-534347856
+  - name: gulp
+    command: >
+      cd $GITPOD_REPO_ROOT/frontend/;
+      yarn && gp sync-done 'frontend-yarn';
+      npx gulp --theme=${PENPOT_THEME} watch
+
+  - name: frontend shadow watch
+    command: >
+      cd $GITPOD_REPO_ROOT/frontend/;
+      gp sync-await 'frontend-yarn';
+      npx shadow-cljs watch main
+
+  - init: gp await-port 5432 && psql -f $GITPOD_REPO_ROOT/docker/gitpod/files/postgresql_init.sql
+    name: backend
+    command: >
+      cd $GITPOD_REPO_ROOT/backend/;
+      ./scripts/start-dev
+
+  - name: exporter shadow watch
+    command:
+      cd $GITPOD_REPO_ROOT/exporter/;
+      gp sync-await 'frontend-yarn';
+      yarn && npx shadow-cljs watch main
+
+  - name: exporter web server
+    command: >
+      cd $GITPOD_REPO_ROOT/exporter/;
+      ./scripts/wait-and-start.sh
+
+  - name: signed terminal
+    before: >
+      [[ ! -z ${GNUGPG}  ]] &&
+      cd ~ &&
+      rm -rf .gnupg &&
+      echo ${GNUGPG} | base64 -d | tar --no-same-owner -xzvf -
+    init: >
+      [[ ! -z ${GNUGPG_KEY}  ]] &&
+      git config --global commit.gpgsign true &&
+      git config --global user.signingkey ${GNUGPG_KEY}
+    command: cd $GITPOD_REPO_ROOT
+
+  - name: redis
+    command: redis-server
+
+  - before: go get github.com/mailhog/MailHog
+    name: mailhog
+    command: MailHog
+
+  - name: Nginx
+    command: >
+      nginx &&
+      multitail /var/log/nginx/access.log -I /var/log/nginx/error.log

CHANGES.md

@@ -0,0 +1,384 @@
+# CHANGELOG #
+
+## :rocket: Next
+
+### :sparkles: New features
+
+### :bug: Bugs fixed
+### :arrow_up: Deps updates
+### :boom: Breaking changes
+### :heart: Community contributions by (Thank you!)
+
+## 1.6.4-alpha
+
+### :sparkles: Minor improvements
+
+-  Decrease default bulk buffers on storage tasks.
+-  Reduce file_change preserve interval to 24h.
+
+### :bug: Bugs fixed
+
+- Don't allow rename drafts project.
+- Fix custom font deletion task.
+- Fix custom font rendering on exporting shapes.
+- Fix font loading on viewer app.
+- Fix problem when moving files with drag & drop.
+- Fix unexpected exception on searching without term.
+- Properly handle nil values on `update-shapes` function.
+- Replace frame term usage by artboard on viewer app.
+
+
+## 1.6.3-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with merge and join nodes [#990](https://github.com/penpot/penpot/issues/990)
+- Fix problem with empty path editing.
+- Fix problem with create component.
+- Fix problem with move-objects.
+- Fix problem with merge and join nodes.
+
+## 1.6.2-alpha
+
+### :bug: Bugs fixed
+
+- Add better auth module logging.
+- Add missing `email` scope to OIDC backend.
+- Add missing cause prop on error loging.
+- Fix empty font-family handling on custom fonts page.
+- Fix incorrect unicode code points handling on draft-to-penpot conversion.
+- Fix some problems with paths.
+- Fix unexpected exception on duplicate project.
+- Fix unexpected exception when user leaves typography name empty.
+- Improve error report on uploading invalid image to library.
+- Minor fix on previous commit.
+- Minor improvements on svg uploading on libraries.
+
+
+## 1.6.1-alpha
+
+### :bug: Bugs fixed
+
+- Add safety check on reg-objects change impl.
+- Fix custom fonts embbedding issue.
+- Fix dashboard ordering issue.
+- Fix problem when creating a component with empty data.
+- Fix problem with moving shapes into frames.
+- Fix problems with mov-objects.
+- Fix unexpected excetion related to rounding integers.
+- Fix wrong type usage on libraries changes.
+- Improve editor lifecycle management.
+- Make the navigation async by default.
+
+
+## 1.6.0-alpha
+
+### :sparkles: New features
+
+- Add improved workspace font selector [Taiga US #292](https://tree.taiga.io/project/penpot/us/292).
+- Add option to interactively scale text [Taiga #1527](https://tree.taiga.io/project/penpot/us/1527)
+- Add performance improvements on dashboard data loading.
+- Add performance improvements to indexes handling on workspace.
+- Add the ability to upload/use custom fonts (and automatically generate all needed webfonts) [Taiga US #292](https://tree.taiga.io/project/penpot/us/292).
+- Transform shapes to path on double click
+- Translate automatic names of new files and projects.
+- Use shift instead of ctrl/cmd to keep aspect ratio [Taiga 1697](https://tree.taiga.io/project/penpot/issue/1697).
+- New translations: Portuguese (Brazil) and Romanias. 
+
+
+### :bug: Bugs fixed
+
+- Remove interactions when the destination artboard is deleted [Taiga #1656](https://tree.taiga.io/project/penpot/issue/1656).
+- Fix problem with fonts that ends with numbers [#940](https://github.com/penpot/penpot/issues/940).
+- Fix problem with imported SVG on editing paths [#971](https://github.com/penpot/penpot/issues/971)
+- Fix problem with color picker positioning
+- Fix order on color palette [#961](https://github.com/penpot/penpot/issues/961)
+- Fix issue when group creation leaves an empty group [#1724](https://tree.taiga.io/project/penpot/issue/1724)
+- Fix problem with :multiple for colors and typographies [#1668](https://tree.taiga.io/project/penpot/issue/1668)
+- Fix problem with locked shapes when change parents [#974](https://github.com/penpot/penpot/issues/974)
+- Fix problem with new nodes in paths [#978](https://github.com/penpot/penpot/issues/978)
+
+### :arrow_up: Deps updates
+
+- Update exporter dependencies (puppeteer), that fixes some unexpected exceptions.
+- Update string manipulation library.
+
+
+### :boom: Breaking changes
+
+- The OIDC setting `PENPOT_OIDC_SCOPES` has changed the default semantics. Before this
+  configuration added scopes to the default set. Now it replaces it, so use with care, because
+  penpot requires at least `name` and `email` props found on the user info object.
+
+### :heart: Community contributions by (Thank you!)
+
+- Translations: Portuguese (Brazil) and Romanias.
+
+
+## 1.5.4-alpha
+
+### :bug: Bugs fixed
+
+- Fix issues on group rendering.
+- Fix problem with text editing auto-height [Taiga #1683](https://tree.taiga.io/project/penpot/issue/1683)
+
+
+## 1.5.3-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem undo/redo.
+
+## 1.5.2-alpha
+
+### :bug: Bugs fixed
+
+- Fix problem with `close-path` command [#917](https://github.com/penpot/penpot/issues/917)
+- Fix wrong query for obtain the profile default project-id
+- Fix problems with empty paths and shortcuts [#923](https://github.com/penpot/penpot/issues/923)
+
+## 1.5.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix issue with bitmap image clipboard.
+- Fix issue when removing all path points.
+- Increase default team invitation token expiration to 48h.
+- Fix wrong error message when an expired token is used.
+
+
+## 1.5.0-alpha
+
+### :sparkles: New features
+
+- Add integration with gitpod.io (an online IDE) [#807](https://github.com/penpot/penpot/pull/807)
+- Allow basic math operations in inputs [Taiga 1383](https://tree.taiga.io/project/penpot/us/1383)
+- Autocomplete color names in hex inputs [Taiga 1596](https://tree.taiga.io/project/penpot/us/1596)
+- Allow to group assets (components and graphics) [Taiga #1289](https://tree.taiga.io/project/penpot/us/1289)
+- Change icon of pinned projects [Taiga 1298](https://tree.taiga.io/project/penpot/us/1298)
+- Internal: refactor of http client, replace internal xhr usage with more modern Fetch API.
+- New features for paths: snap points on edition, add/remove nodes, merge/join/split nodes. [Taiga #907](https://tree.taiga.io/project/penpot/us/907)
+- Add OpenID-Connect support.
+- Reimplement social auth providers on top of the generic openid impl.
+
+### :bug: Bugs fixed
+
+- Fix problem with pan and space [#811](https://github.com/penpot/penpot/issues/811)
+- Fix issue when parsing exponential numbers in paths
+- Remove legacy system user and team [#843](https://github.com/penpot/penpot/issues/843)
+- Fix ordering of copy pasted objects [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1617)
+- Fix problems with blending modes [#837](https://github.com/penpot/penpot/issues/837)
+- Fix problem with zoom an selection rect [#845](https://github.com/penpot/penpot/issues/845)
+- Fix problem displaying team statistics [#859](https://github.com/penpot/penpot/issues/859)
+- Fix problems with text editor selection [Taiga #1546](https://tree.taiga.io/project/penpot/issue/1546)
+- Fix problem when opening the context menu in dashboard at the bottom [#856](https://github.com/penpot/penpot/issues/856)
+- Fix problem when clicking an interactive group in view mode [#863](https://github.com/penpot/penpot/issues/863)
+- Fix visibility of pages in sitemap when changing page [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1618)
+- Fix visual problem with group invite [Taiga #1290](https://tree.taiga.io/project/penpot/issue/1290)
+- Fix issues with promote owner panel [Taiga #763](https://tree.taiga.io/project/penpot/issue/763)
+- Allow use library colors when defining gradients [Taiga #1614](https://tree.taiga.io/project/penpot/issue/1614)
+- Fix group selrect not updating after alignment [#895](https://github.com/penpot/penpot/issues/895)
+
+### :arrow_up: Deps updates
+
+### :boom: Breaking changes
+
+- Translations refactor: now penpot uses gettext instead of a custom
+  JSON, and each locale has its own separated file. All translations
+  should be contributed via the weblate.org service.
+
+### :heart: Community contributions by (Thank you!)
+
+- madmath03 (by [Monogramm](https://github.com/Monogramm)) [#807](https://github.com/penpot/penpot/pull/807)
+- zzkt [#814](https://github.com/penpot/penpot/pull/814)
+
+
+## 1.4.1-alpha
+
+### :bug: Bugs fixed
+
+- Fix typography unlinking.
+- Fix incorrect measures on shapes outside artboard.
+- Fix issues on svg parsing related to numbers with exponents.
+- Fix some race conditions on removing shape from workspace.
+- Fix incorrect state management of user lang selection.
+- Fix email validation usability issue on team invitation lightbox.
+
+
+## 1.4.0-alpha
+
+### :sparkles: New features
+
+- Add blob-encoding v3 (uses ZSTD+transit) [#738](https://github.com/penpot/penpot/pull/738)
+- Add http caching layer on top of Query RPC.
+- Add layer opacity and blend mode to shapes [Taiga #937](https://tree.taiga.io/project/penpot/us/937)
+- Add more chinese translations [#726](https://github.com/penpot/penpot/pull/726)
+- Add native support for text-direction (RTL, LTR & auto).
+- Add several enhancements in shape selection [Taiga #1195](https://tree.taiga.io/project/penpot/us/1195)
+- Add thumbnail in memory caching mechanism.
+- Add turkish translation strings [#759](https://github.com/penpot/penpot/pull/759), [#794](https://github.com/penpot/penpot/pull/794)
+- Duplicate and move files and projects [Taiga #267](https://tree.taiga.io/project/penpot/us/267)
+- Hide viewer navbar on fullscreen [Taiga 1375](https://tree.taiga.io/project/penpot/us/1375)
+- Import SVG will create Penpot's shapes [Taiga #1006](https://tree.taiga.io/project/penpot/us/1066)
+- Improve french translations [#731](https://github.com/penpot/penpot/pull/731)
+- Reimplement workspace presence (remove database state).
+- Remember last visited team when you re-enter the application [Taiga #1376](https://tree.taiga.io/project/penpot/us/1376)
+- Rename artboard with double click on the title [Taiga #1392](https://tree.taiga.io/project/penpot/us/1392)
+- Replace Slate-Editor with DraftJS [Taiga #1346](https://tree.taiga.io/project/penpot/us/1346)
+- Set proper page title [Taiga #1377](https://tree.taiga.io/project/penpot/us/1377)
+
+
+### :bug: Bugs fixed
+
+- Disable buttons in view mode for users without permissions [Taiga #1328](https://tree.taiga.io/project/penpot/issue/1328)
+- Fix broken profile and profile options form.
+- Fix calculate size of some animated gifs [Taiga #1487](https://tree.taiga.io/project/penpot/issue/1487)
+- Fix error with the "Navigate to" button on prototypes [Taiga #1268](https://tree.taiga.io/project/penpot/issue/1268)
+- Fix issue when undo after changing the artboard of a shape [Taiga #1304](https://tree.taiga.io/project/penpot/issue/1304)
+- Fix issue with Alt key in distance measurement [#672](https://github.com/penpot/penpot/issues/672)
+- Fix issue with blending modes in masks [Taiga #1476](https://tree.taiga.io/project/penpot/issue/1476)
+- Fix issue with blocked shapes [Taiga #1480](https://tree.taiga.io/project/penpot/issue/1480)
+- Fix issue with comments styles on dashboard [Taiga #1405](https://tree.taiga.io/project/penpot/issue/1405)
+- Fix issue with default square grid [Taiga #1344](https://tree.taiga.io/project/penpot/issue/1344)
+- Fix issue with enter key shortcut [#775](https://github.com/penpot/penpot/issues/775)
+- Fix issue with enter to edit paths [Taiga #1481](https://tree.taiga.io/project/penpot/issue/1481)
+- Fix issue with mask and flip [#715](https://github.com/penpot/penpot/issues/715)
+- Fix issue with masks interactions outside bounds [#718](https://github.com/penpot/penpot/issues/718)
+- Fix issue with middle mouse button press moving the canvas when not moving mouse [#717](https://github.com/penpot/penpot/issues/717)
+- Fix issue with resolved comments [Taiga #1406](https://tree.taiga.io/project/penpot/issue/1406)
+- Fix issue with rotated blur [Taiga #1370](https://tree.taiga.io/project/penpot/issue/1370)
+- Fix issue with rotation degree input [#741](https://github.com/penpot/penpot/issues/741)
+- Fix issue with system shortcuts and application [#737](https://github.com/penpot/penpot/issues/737)
+- Fix issue with team management in dashboard [Taiga #1475](https://tree.taiga.io/project/penpot/issue/1475)
+- Fix issue with typographies panel cannot be collapsed [#707](https://github.com/penpot/penpot/issues/707)
+- Fix text selection in comments [#745](https://github.com/penpot/penpot/issues/745)
+- Update Work-Sans font [#744](https://github.com/penpot/penpot/issues/744)
+- Fix issue with recent files not showing [Taiga #1493](https://tree.taiga.io/project/penpot/issue/1493)
+- Fix issue when promoting to owner [Taiga #1494](https://tree.taiga.io/project/penpot/issue/1494)
+- Fix cannot click on blocked elements in viewer [Taiga #1430](https://tree.taiga.io/project/penpot/issue/1430)
+- Fix SVG not showing properties at code [Taiga #1437](https://tree.taiga.io/project/penpot/issue/1437)
+- Fix shadows when exporting groups [Taiga #1495](https://tree.taiga.io/project/penpot/issue/1495)
+- Fix drag-select when renaming layer text [Taiga #1307](https://tree.taiga.io/project/penpot/issue/1307)
+- Fix layout problem for editable selects [Taiga #1488](https://tree.taiga.io/project/penpot/issue/1488)
+- Fix artboard title wasn't move when resizing [Taiga #1479](https://tree.taiga.io/project/penpot/issue/1479)
+- Fix titles in viewer thumbnails too long [Taiga #1474](https://tree.taiga.io/project/penpot/issue/1474)
+- Fix when right click on a selected text shows artboard contextual menu [Taiga #1226](https://tree.taiga.io/project/penpot/issue/1226)
+
+### :boom: Breaking changes
+
+- The LDAP configuration variables interpolation starts using `:`
+  (example `:username`) instead of `$`. The main reason is avoid
+  unnecesary conflict with bash interpolation.
+
+
+### :arrow_up: Deps updates
+
+- Update backend to JDK16.
+- Update exporter nodejs to v14.16.0
+
+
+### :heart: Community contributions by (Thank you!)
+
+- iblueer [#726](https://github.com/penpot/penpot/pull/726)
+- gizembln [#759](https://github.com/penpot/penpot/pull/759)
+- girafic [#748](https://github.com/penpot/penpot/pull/748)
+- mbrksntrk [#794](https://github.com/penpot/penpot/pull/794)
+
+
+## 1.3.0-alpha
+
+### :sparkles: New features
+
+- Add emailcatcher and ldap test containers to devenv. [#506](https://github.com/penpot/penpot/pull/506)
+- Add major refactor of internal pubsub/redis code; improves scalability and performance [#640](https://github.com/penpot/penpot/pull/640)
+- Add more chinese transtions [#687](https://github.com/penpot/penpot/pull/687)
+- Add more presets for artboard [#654](https://github.com/penpot/penpot/pull/654)
+- Add optional loki integration [#645](https://github.com/penpot/penpot/pull/645)
+- Add proper http session lifecycle handling.
+- Allow to set border radius of each rect corner individually
+- Bounce & Complaint handling [#635](https://github.com/penpot/penpot/pull/635)
+- Disable groups interactions when holding "Ctrl" key (deep selection)
+- New action in context menu to "edit" some shapes (binded to key "Enter")
+
+
+### :bug: Bugs fixed
+
+- Add more improvements to french translation strings [#591](https://github.com/penpot/penpot/pull/591)
+- Add some missing database indexes (mainly improves performance on large databases on file-update rpc method, and some background tasks).
+- Disables filters in masking elements (issue with Firefox rendering)
+- Drawing tool will have priority over resize/rotate handlers [Taiga #1225](https://tree.taiga.io/project/penpot/issue/1225)
+- Fix broken bounding box on editing paths [Taiga #1254](https://tree.taiga.io/project/penpot/issue/1254)
+- Fix corner cases on invitation/signup flows.
+- Fix errors on onboarding file [Taiga #1287](https://tree.taiga.io/project/penpot/issue/1287)
+- Fix infinite recursion on logout.
+- Fix issues with frame selection [Taiga #1300](https://tree.taiga.io/project/penpot/issue/1300), [Taiga #1255](https://tree.taiga.io/project/penpot/issue/1255)
+- Fix local fonts error [#691](https://github.com/penpot/penpot/issues/691)
+- Fix issue width handoff code generation [Taiga #1204](https://tree.taiga.io/project/penpot/issue/1204)
+- Fix issue with indices refreshing on page changes [#646](https://github.com/penpot/penpot/issues/646)
+- Have language change notification written in the new language [Taiga #1205](https://tree.taiga.io/project/penpot/issue/1205)
+- Hide register screen when registration is disabled [#598](https://github.com/penpot/penpot/issues/598)
+- Properly handle errors on github, gitlab and ldap auth backends.
+- Properly mark profile auth backend (on first register/ auth with 3rd party auth provider).
+- Refactor LDAP auth backend.
+
+
+### :heart: Community contributions by (Thank you!)
+
+- girafic [#538](https://github.com/penpot/penpot/pull/654)
+- arkhi [#591](https://github.com/penpot/penpot/pull/591)
+
+
+## 1.2.0-alpha
+
+### :sparkles: New features
+
+- Add horizontal/vertical flip
+- Add images lock proportions by default [#541](https://github.com/penpot/penpot/discussions/541), [#609](https://github.com/penpot/penpot/issues/609)
+- Add new blob storage format (Zstd+nippy)
+- Add user feedback form
+- Improve French translations
+- Improve component testing
+- Increase default deletion delay to 7 days
+- Show a pixel grid when zoom greater than 800% [#519](https://github.com/penpot/penpot/discussions/519)
+- Fix behavior of select all command when there are objects outside frames [Taiga #1209](https://tree.taiga.io/project/penpot/issue/1209)
+
+
+### :bug: Bugs fixed
+
+- Fix 404 when access shared link [#615](https://github.com/penpot/penpot/issues/615)
+- Fix 500 when requestion password reset
+- Fix issue when transforming path shapes [Taiga #1170](https://tree.taiga.io/project/penpot/issue/1170)
+- Fix apply a color to a text selection from color palette was not working [Taiga #1189](https://tree.taiga.io/project/penpot/issue/1189)
+- Fix issues when moving shapes outside groups [Taiga #1138](https://tree.taiga.io/project/penpot/issue/1138)
+- Fix ldap function called on login click
+- Fix logo icon in viewer should go to dashboard [Taiga #1149](https://tree.taiga.io/project/penpot/issue/1149)
+- Fix ordering when restoring deleted shapes in sync [Taiga #1163](https://tree.taiga.io/project/penpot/issue/1163)
+- Fix issue when editing text immediately after creating [Taiga #1207](https://tree.taiga.io/project/penpot/issue/1207)
+- Fix issue when pasting URL's copied from the browser url bar [Taiga #1187](https://tree.taiga.io/project/penpot/issue/1187)
+- Fix issue with multiple selection and groups [Taiga #1128](https://tree.taiga.io/project/penpot/issue/1128)
+- Fix issue with red handler indicator on resize [Taiga #1188](https://tree.taiga.io/project/penpot/issue/1188)
+- Fix show correct error when google auth is disabled [Taiga #1119](https://tree.taiga.io/project/penpot/issue/1119)
+- Fix text alignment in preview [#594](https://github.com/penpot/penpot/issues/594)
+- Fix unexpected exception when uploading image [Taiga #1120](https://tree.taiga.io/project/penpot/issue/1120)
+- Fix updates on collaborative editing not updating selection rectangles [Taiga #1127](https://tree.taiga.io/project/penpot/issue/1127)
+- Make the team deletion deferred (in the same way other objects)
+
+### :heart: Community contributions by (Thank you!)
+
+- abtinmo [#538](https://github.com/penpot/penpot/pull/538)
+- kdrag0n [#585](https://github.com/penpot/penpot/pull/585)
+- nisrulz [#586](https://github.com/penpot/penpot/pull/586)
+- tomer [#575](https://github.com/penpot/penpot/pull/575)
+- violoncelloCH [#554](https://github.com/penpot/penpot/pull/554)
+
+## 1.1.0-alpha
+
+- Bugfixing and stabilization post-launch
+- Some changes to the register flow
+- Improved MacOS shortcuts and helpers
+- Small changes to shape creation
+
+
+## 1.0.0-alpha
+
+Initial release

CONTRIBUTING.md

@@ -1,8 +1,8 @@
 # Contributing Guide #
 
 Thank you for your interest in contributing to Penpot. This is a
-generic guide that details how to contribute to Penpot in a way that is
-efficient for everyone. If you want a specific documentation for
+generic guide that details how to contribute to Penpot in a way that
+is efficient for everyone. If you want a specific documentation for
 different parts of the platform, please refer to `docs/` directory.
 
 
@@ -19,12 +19,20 @@ If you found a bug, please report it, as far as possible with:
 - a browser and the browser version used
 - a dev tools console exception stack trace (if it is available)
 
+If you found a bug that you consider better discuse in private (for
+example: security bugs), consider first send an email to
+`info@penpot.app`.
+
+**We don't have formal bug bounty program for security reports; this
+is an open source application and your contribution will be recognized
+in the changelog.**
+
 
 ## Pull requests ##
 
 If you want propose a change or bug fix with the Pull-Request system
-firstly you should carefully read the **Contributor License Aggreement**
-section and format your commits accordingly.
+firstly you should carefully read the **DCO** section and format your
+commits accordingly.
 
 If you intend to fix a bug it's fine to submit a pull request right
 away but we still recommend to file an issue detailing what you're
@@ -127,7 +135,7 @@ This Code of Conduct is adapted from the Contributor Covenant, version
 1.1.0, available from http://contributor-covenant.org/version/1/1/0/
 
 
-## Contributor License Agreement ##
+## Developer's Certificate of Origin (DCO) ##
 
 By submitting code you are agree and can certify the below:
 
@@ -157,9 +165,9 @@ By submitting code you are agree and can certify the below:
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
 
-Then, all your patches should contain a sign-off at the end of the
-patch/commit description body. It can be automatically added on adding
-`-s` parameter to `git commit`.
+Then, all your code patches (**documentation are excluded**) should
+contain a sign-off at the end of the patch/commit description body. It
+can be automatically added on adding `-s` parameter to `git commit`.
 
 This is an example of the aspect of the line:
 

README.md

@@ -3,32 +3,21 @@
 [uri_license_image]: https://img.shields.io/badge/MPL-2.0-blue.svg
 
 [![License: MPL-2.0][uri_license_image]][uri_license]
-[![Managed with Taiga.io](https://img.shields.io/badge/managed%20with-TAIGA.io-709f14.svg)](https://tree.taiga.io/project/uxbox/ "Managed with Taiga.io")
+[![Gitter](https://badges.gitter.im/sereno-xyz/community.svg)](https://gitter.im/penpot/community)
+[![Managed with Taiga.io](https://img.shields.io/badge/managed%20with-TAIGA.io-709f14.svg)](https://tree.taiga.io/project/penpot/ "Managed with Taiga.io")
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/penpot/penpot)
 
 
 # PENPOT #
 
-We’re excited to share that Uxbox is now Penpot! We’re changing the name, but keeping the same project essence. Stay in the loop for more news coming early 2021. Alpha release is close!
+Penpot is the first Open Source design and prototyping platform meant
+for cross-domain teams. Non dependent on operating systems, Penpot is
+web based and works with open web standards (SVG). For all and
+empowered by the community.
 
-![PENPOT](https://raw.githubusercontent.com/penpot/penpot/develop/docs/screenshot.png)
+![PENPOT](https://penpot.app/images/workspace-ui.jpg)
 
 
-## Introduction ##
-
-The open-source solution for design and prototyping. PENPOT is
-currently at an early development stage but we are working hard to
-bring you the beta version as soon as possible. Follow the project
-progress in Twitter or Github and stay tuned!
-
-
-## SVG based ##
-
-Penpot works with SVG, a standard format, for all your designs and
-prototypes . This means that all your stuff in Penpot is portable and
-editable in many other vector tools and easy to use on the web.
-
-[See SVG specification](https://www.w3.org/Graphics/SVG/)
-
 ## Contributing ##
 
 **Open to you!**
@@ -40,10 +29,17 @@ and improve Penpot. All your awesome ideas and code are welcome!
 Please refer to the [Contributing Guide](./CONTRIBUTING.md)
 
 
+## Documentation ##
+
+Please refer to the [help center](https://help.penpot.app).
+
+
 ## License ##
 
 ```
 This Source Code Form is subject to the terms of the Mozilla Public
 License, v. 2.0. If a copy of the MPL was not distributed with this
 file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Copyright (c) UXBOX Labs SL
 ```

backend/deps.edn

@@ -3,45 +3,57 @@
   "clojars" {:url "https://clojars.org/repo"}
   "jcenter" {:url "https://jcenter.bintray.com/"}}
  :deps
- {org.clojure/clojure {:mvn/version "1.10.1"}
-  org.clojure/clojurescript {:mvn/version "1.10.773"}
-  org.clojure/data.json {:mvn/version "1.0.0"}
-  org.clojure/core.async {:mvn/version "1.3.610"}
+ {org.clojure/clojure {:mvn/version "1.10.3"}
+  org.clojure/data.json {:mvn/version "2.2.3"}
+  org.clojure/core.async {:mvn/version "1.3.618"}
+  org.clojure/tools.cli {:mvn/version "1.0.206"}
+  org.clojure/clojurescript {:mvn/version "1.10.844"}
 
   ;; Logging
   org.clojure/tools.logging {:mvn/version "1.1.0"}
-  org.apache.logging.log4j/log4j-api {:mvn/version "2.13.3"}
-  org.apache.logging.log4j/log4j-core {:mvn/version "2.13.3"}
-  org.apache.logging.log4j/log4j-web {:mvn/version "2.13.3"}
-  org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.3"}
-  org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.3"}
-  org.slf4j/slf4j-api {:mvn/version "1.7.30"}
+  org.apache.logging.log4j/log4j-api {:mvn/version "2.14.1"}
+  org.apache.logging.log4j/log4j-core {:mvn/version "2.14.1"}
+  org.apache.logging.log4j/log4j-web {:mvn/version "2.14.1"}
+  org.apache.logging.log4j/log4j-jul {:mvn/version "2.14.1"}
+  org.apache.logging.log4j/log4j-slf4j18-impl {:mvn/version "2.14.1"}
+  org.slf4j/slf4j-api {:mvn/version "2.0.0-alpha1"}
+  org.zeromq/jeromq {:mvn/version "0.5.2"}
 
+  com.taoensso/nippy {:mvn/version "3.1.1"}
+  com.github.luben/zstd-jni {:mvn/version "1.4.9-5"}
+
+  ;; NOTE: don't upgrade to latest version, breaking change is
+  ;; introduced on 0.10.0 that suffixes counters with _total if they
+  ;; are not already has this suffix.
   io.prometheus/simpleclient {:mvn/version "0.9.0"}
   io.prometheus/simpleclient_hotspot {:mvn/version "0.9.0"}
+  io.prometheus/simpleclient_jetty {:mvn/version "0.9.0"
+                                    :exclusions [org.eclipse.jetty/jetty-server
+                                                 org.eclipse.jetty/jetty-servlet]}
   io.prometheus/simpleclient_httpserver {:mvn/version "0.9.0"}
 
-  selmer/selmer {:mvn/version "1.12.28"}
-  expound/expound {:mvn/version "0.8.5"}
+  selmer/selmer {:mvn/version "1.12.40"}
+  expound/expound {:mvn/version "0.8.9"}
   com.cognitect/transit-clj {:mvn/version "1.0.324"}
 
-  io.lettuce/lettuce-core {:mvn/version "5.2.2.RELEASE"}
-  java-http-clj/java-http-clj {:mvn/version "0.4.1"}
+  io.lettuce/lettuce-core {:mvn/version "6.1.2.RELEASE"}
+  java-http-clj/java-http-clj {:mvn/version "0.4.2"}
 
-  info.sunng/ring-jetty9-adapter {:mvn/version "0.14.0"}
-  seancorfield/next.jdbc {:mvn/version "1.1.588"}
-  metosin/reitit-ring {:mvn/version "0.5.5"}
-  org.postgresql/postgresql {:mvn/version "42.2.16"}
-  com.zaxxer/HikariCP {:mvn/version "3.4.5"}
+  info.sunng/ring-jetty9-adapter {:mvn/version "0.15.1"}
+  com.github.seancorfield/next.jdbc {:mvn/version "1.2.659"}
+  metosin/reitit-ring {:mvn/version "0.5.13"}
+  metosin/jsonista {:mvn/version "0.3.3"}
 
-  funcool/log4j2-clojure {:mvn/version "2020.11.23-1"}
-  funcool/datoteka {:mvn/version "1.2.0"}
-  funcool/promesa {:mvn/version "5.1.0"}
-  funcool/cuerdas {:mvn/version "2020.03.26-3"}
+  org.postgresql/postgresql {:mvn/version "42.2.20"}
+  com.zaxxer/HikariCP {:mvn/version "4.0.3"}
 
-  buddy/buddy-core {:mvn/version "1.9.0"}
-  buddy/buddy-hashers {:mvn/version "1.7.0"}
-  buddy/buddy-sign {:mvn/version "3.3.0"}
+  funcool/datoteka {:mvn/version "2.0.0"}
+  funcool/promesa {:mvn/version "6.0.1"}
+  funcool/cuerdas {:mvn/version "2021.05.09-0"}
+
+  buddy/buddy-core {:mvn/version "1.10.1"}
+  buddy/buddy-hashers {:mvn/version "1.8.1"}
+  buddy/buddy-sign {:mvn/version "3.4.1"}
 
   lambdaisland/uri {:mvn/version "1.4.54"
                     :exclusions [org.clojure/data.json]}
@@ -51,54 +63,41 @@
   org.jsoup/jsoup {:mvn/version "1.13.1"}
   org.im4java/im4java {:mvn/version "1.4.0"}
   org.lz4/lz4-java {:mvn/version "1.7.1"}
-  com.github.spullara.mustache.java/compiler {:mvn/version "0.9.6"}
   commons-io/commons-io {:mvn/version "2.8.0"}
-  com.draines/postal {:mvn/version "2.0.3"
-                      :exclusions [commons-codec/commons-codec]}
+  com.sun.mail/jakarta.mail {:mvn/version "2.0.1"}
 
-  puppetlabs/clj-ldap {:mvn/version"0.3.0"}
+  org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
+  integrant/integrant {:mvn/version "0.8.0"}
+
+  software.amazon.awssdk/s3 {:mvn/version "2.16.62"}
 
   ;; exception printing
   io.aviso/pretty {:mvn/version "0.1.37"}
-
-  mount/mount {:mvn/version "0.1.16"}
   environ/environ {:mvn/version "1.2.0"}}
  :paths ["src" "resources" "../common" "common"]
  :aliases
  {:dev
   {:extra-deps
-   {com.bhauman/rebel-readline {:mvn/version "0.1.4"}
-    org.clojure/tools.namespace {:mvn/version "1.0.0"}
-    org.clojure/test.check {:mvn/version "1.0.0"}
-    clj-kondo/clj-kondo {:mvn/version "RELEASE"}
+   {com.bhauman/rebel-readline {:mvn/version "RELEASE"}
+    org.clojure/tools.namespace {:mvn/version "RELEASE"}
+    org.clojure/test.check {:mvn/version "RELEASE"}
 
-    fipp/fipp {:mvn/version "0.6.21"}
-    criterium/criterium {:mvn/version "0.4.5"}
+    fipp/fipp {:mvn/version "0.6.23"}
+    criterium/criterium {:mvn/version "0.4.6"}
     mockery/mockery {:mvn/version "0.1.4"}}
-   :extra-paths ["tests"]}
-
-  ;; :fn-media-loader
-  ;; {:exec-fn app.cli.media-loader/run
-  ;;  :args {}}
+   :extra-paths ["tests" "dev"]}
 
   :fn-fixtures
   {:exec-fn app.cli.fixtures/run
    :args {}}
 
-  :lint
-  {:main-opts ["-m" "clj-kondo.main"]}
-
   :tests
-  {:extra-deps {lambdaisland/kaocha {:mvn/version "0.0-581"}}
+  {:extra-deps {lambdaisland/kaocha {:mvn/version "1.0.829"}}
    :main-opts ["-m" "kaocha.runner"]}
 
   :outdated
-  {:extra-deps {olical/depot {:mvn/version "1.8.4"}}
-   :main-opts ["-m" "depot.outdated.main"]}
-
-  :jar
-  {:extra-deps {seancorfield/depstar {:mvn/version "RELEASE"}}
-   :main-opts ["-m" "hf.depstar.jar" "-S" "target/app.jar"]}
+  {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
+   :main-opts ["-m" "antq.core"]}
 
   :jmx-remote
   {:jvm-opts ["-Dcom.sun.management.jmxremote"

backend/dev/user.clj

@@ -2,30 +2,32 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) UXBOX Labs SL
 
 (ns user
   (:require
+   [app.common.exceptions :as ex]
+   [app.config :as cfg]
+   [app.main :as main]
+   [app.util.blob :as blob]
+   [app.util.json :as json]
+   [app.util.time :as dt]
+   [app.util.transit :as t]
+   [clojure.java.io :as io]
+   [clojure.pprint :refer [pprint print-table]]
+   [clojure.repl :refer :all]
    [clojure.spec.alpha :as s]
+   [clojure.spec.gen.alpha :as sgen]
+   [clojure.test :as test]
+   [clojure.test :as test]
    [clojure.tools.namespace.repl :as repl]
    [clojure.walk :refer [macroexpand-all]]
-   [clojure.pprint :refer [pprint]]
-   [clojure.test :as test]
-   [clojure.java.io :as io]
-   [app.common.pages :as cp]
-   [clojure.repl :refer :all]
    [criterium.core :refer [quick-bench bench with-progress-reporting]]
-   [clj-kondo.core :as kondo]
-   [app.migrations]
-   [app.db :as db]
-   [app.metrics :as mtx]
-   [app.util.storage :as st]
-   [app.util.time :as tm]
-   [app.util.blob :as blob]
-   [mount.core :as mount]))
+   [integrant.core :as ig]))
+
+(repl/disable-reload! (find-ns 'integrant.core))
+
+(defonce system nil)
 
 ;; --- Benchmarking Tools
 
@@ -47,20 +49,6 @@
 
 ;; --- Development Stuff
 
-(defn- start
-  []
-  (-> #_(mount/except #{#'app.scheduled-jobs/scheduler})
-      (mount/start)))
-
-(defn- stop
-  []
-  (mount/stop))
-
-(defn restart
-  []
-  (stop)
-  (repl/refresh :after 'user/start))
-
 (defn- run-tests
   ([] (run-tests #"^app.tests.*"))
   ([o]
@@ -75,16 +63,35 @@
            (test/test-vars [(resolve o)]))
        (test/test-ns o)))))
 
-(defn lint
-  ([] (lint ""))
-  ([path]
-   (-> (kondo/run!
-        {:lint [(str "src/" path)]
-         :cache false
-         :config {:linters
-                  {:unresolved-symbol
-                   {:exclude ['(app.services.mutations/defmutation)
-                              '(app.services.queries/defquery)
-                              '(app.db/with-atomic)
-                              '(promesa.core/let)]}}}})
-       (kondo/print!))))
+(defn- start
+  []
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             (-> main/system-config
+                                 (ig/prep)
+                                 (ig/init))))
+  :started)
+
+(defn- stop
+  []
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             nil))
+  :stoped)
+
+(defn restart
+  []
+  (stop)
+  (repl/refresh :after 'user/start))
+
+(defn restart-all
+  []
+  (stop)
+  (repl/refresh-all :after 'user/start))
+
+(defn compression-bench
+  [data]
+  (print-table
+   [{:v1 (alength (blob/encode data {:version 1}))
+     :v2 (alength (blob/encode data {:version 2}))
+     :v3 (alength (blob/encode data {:version 3}))}]))

backend/resources/builtin.edn

@@ -1,11 +0,0 @@
-{:icons
- [{:name "Material Design (Action)"
-   :path "./material/action/svg/production"
-   :regex #"^.*_48px\.svg$"}]
-
- :images
- [{:name "Generic Collection 1"
-   :path "./my-images/collection1/"
-   :regex #"^.*\.(png|jpg|webp)$"}]}
-
-

backend/resources/config/default.edn

@@ -1,44 +0,0 @@
-{;; A secret key used for create tokens
- ;; WARNING: this is a default secret key and
- ;; it should be overwritten in production env.
- :secret "5qjiAn-QUpawUNqGP10UZKklSqbLKcdGY3sJpq0UUACpVXGg2HOFJCBejDWVHskhRyp7iHb4rjOLXX2ZjF-5cw"
- 
- :registration
- {
-  :enabled true}
- 
- :smtp
- {:host "localhost" ;; Hostname of the desired SMTP server.
-  :port 25          ;; Port of SMTP server.
-  :user nil         ;; Username to authenticate with (if authenticating).
-  :pass nil         ;; Password to authenticate with (if authenticating).
-  :ssl false        ;; Enables SSL encryption if value is truthy.
-  :tls false        ;; Enables TLS encryption if value is truthy.
-  :enabled false    ;; Enables SMTP if value is truthy.
-  :noop true}
-
- :auth-options {:alg :a256kw :enc :a128cbc-hs256}
-
- :email {:reply-to "no-reply@uxbox.io"
-         :from "no-reply@uxbox.io"
-         :support "support@uxbox.io"}
-
- :http {:port 6060
-        :max-body-size 52428800
-        :debug true}
-
- :media
- {:directory "resources/public/media"
-  :uri "http://localhost:6060/media/"}
-
- :static
- {:directory "resources/public/static"
-  :uri "http://localhost:6060/static/"}
-
- :database
- {:adapter "postgresql"
-  :username nil
-  :password nil
-  :database-name "uxbox"
-  :server-name "localhost"
-  :port-number 5432}}

backend/resources/config/test.edn

@@ -1,18 +0,0 @@
-{:migrations
- {:verbose false}
-
- :media
- {:directory "/tmp/uxbox/media"
-  :uri "http://localhost:6060/media/"}
-
- :static
- {:directory "/tmp/uxbox/static"
-  :uri "http://localhost:6060/static/"}
-
- :database
- {:adapter "postgresql"
-  :username nil
-  :password nil
-  :database-name "test"
-  :server-name "localhost"
-  :port-number 5432}}

backend/resources/emails-mjml/change-email/en.mjml

@@ -30,14 +30,14 @@
           for security reasons.
         </mj-text>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -57,7 +57,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/invite-to-team/en.mjml

@@ -23,14 +23,14 @@
           Accept invite
         </mj-button>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -50,7 +50,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/password-recovery/en.mjml

@@ -32,14 +32,14 @@
           it. Your password won't be changed.
         </mj-text>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -59,7 +59,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails-mjml/register/en.mjml

@@ -21,7 +21,7 @@
       <mj-column>
         <mj-text font-size="24px" font-weight="600">Hello {{name}}!</mj-text>
         <mj-text>
-          Thanks for signing up for your UXBOX account! Please verify your
+          Thanks for signing up for your Penpot account! Please verify your
           email using the link below adn get started building mockups and
           prototypes today!
         </mj-text>
@@ -29,14 +29,14 @@
           Verify email
         </mj-button>
         <mj-text>Enjoy!</mj-text>
-        <mj-text>The UXBOX team.</mj-text>
+        <mj-text>The Penpot team.</mj-text>
       </mj-column>
     </mj-section>
 
     <mj-section padding="24px 0 0 0">
       <mj-column width="425px">
         <mj-text align="center" font-size="14px" color="#64666A">
-          UXBOX is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
+          Penpot is the first Open Source prototyping platform that will be embraced by multidisciplinary teams.
         </mj-text>
       </mj-column>
     </mj-section>
@@ -56,7 +56,7 @@
     <mj-section padding="0 0 24px 0">
       <mj-column>
         <mj-text align="center" font-size="14px" color="#64666A" line-height="150%">
-          UXBOX © 2020 | Made with &lt;3 and Open Source
+          Penpot © 2020 | Made with &lt;3 and Open Source
         </mj-text>
       </mj-column>
     </mj-section>

backend/resources/emails/change-email/en.html

@@ -450,7 +450,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot © 2020 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/change-email/en.txt

@@ -10,4 +10,4 @@ If you received this email by mistake, please consider changing your password
 for security reasons.
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/feedback/en.html

@@ -0,0 +1,45 @@
+<!doctype html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <title></title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  </head>
+  <body>
+    <p>
+      <strong>Feedback from:</strong><br />
+      {% if profile %}
+        <span>
+          <span>Name: </span>
+          <span><code>{{profile.fullname}}</code></span>
+        </span>
+        <br />
+
+        <span>
+          <span>Email: </span>
+          <span>{{profile.email}}</span>
+        </span>
+        <br />
+
+        <span>
+          <span>ID: </span>
+          <span><code>{{profile.id}}</code></span>
+        </span>
+      {% else %}
+        <span>
+          <span>Email: </span>
+          <span>{{profile.email}}</span>
+        </span>
+      {% endif %}
+    </p>
+    <p>
+      <strong>Subject:</strong><br />
+      <span>{{subject}}</span>
+    </p>
+
+    <p>
+      <strong>Message:</strong><br />
+      {{content|linebreaks-br|safe}}
+    </p>
+  </body>
+</html>

backend/resources/emails/feedback/en.subj

@@ -0,0 +1 @@
+[PENPOT FEEDBACK]: {{subject|abbreviate:19}} (from {{email}})

backend/resources/emails/feedback/en.txt

@@ -0,0 +1,9 @@
+{% if profile %}
+Feedback profile: {{profile.fullname}} <{{profile.email}}> / {{profile.id}}
+{% else %}
+Feedback from: {{email}}
+{% endif %}
+
+Subject: {{subject}}
+
+{{content}}

backend/resources/emails/invite-to-team/en.html

@@ -440,7 +440,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot © 2020 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/invite-to-team/en.subj

@@ -1 +1 @@
-Inviation to join {{team}}
+Invitation to join {{team}}

backend/resources/emails/invite-to-team/en.txt

@@ -7,4 +7,4 @@ Accept invitation using this link:
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/password-recovery/en.html

@@ -445,7 +445,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot © 2020 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/password-recovery/en.txt

@@ -9,4 +9,4 @@ If you received this email by mistake, you can safely ignore it. Your password
 won't be changed.
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/emails/register/en.html

@@ -440,7 +440,7 @@
                 <table border="0" cellpadding="0" cellspacing="0" role="presentation" style="vertical-align:top;" width="100%">
                   <tr>
                     <td align="center" style="font-size:0px;padding:10px 25px;word-break:break-word;">
-                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot © 2020 | Made with &lt;3 and Open Source</div>
+                      <div style="font-family:Source Sans Pro, sans-serif;font-size:14px;line-height:150%;text-align:center;color:#64666A;">Penpot @ 2021 | Made with &lt;3 and Open Source</div>
                     </td>
                   </tr>
                 </table>

backend/resources/emails/register/en.txt

@@ -1,9 +1,9 @@
 Hello {{name}}!
 
-Thanks for signing up for your UXBOX account! Please verify your email using the
+Thanks for signing up for your Penpot account! Please verify your email using the
 link below adn get started building mockups and prototypes today!
 
 {{ public-uri }}/#/auth/verify-token?token={{token}}
 
 Enjoy!
-The UXBOX team.
+The Penpot team.

backend/resources/error-report.tmpl

@@ -0,0 +1,194 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>penpot - error report {{id}}</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono">
+    <style>
+      body {
+        margin: 0px;
+        padding: 0px;
+      }
+      pre {
+        margin: 0px;
+      }
+      * {
+        font-family: "JetBrains Mono", monospace; 
+        font-size: 12px;
+      }
+      .table {
+        display: flex;
+        flex-direction: column;
+        margin: 10px;
+      }
+
+      .table-row {
+        display: flex;
+        /* width: 100%; */
+        /* border: 1px solid red; */
+      }
+
+      .table-key {
+        font-weight: 600;
+        width: 60px;
+        padding: 4px;
+      }
+
+      .table-val {
+        font-weight: 200;
+        color: #333;
+        padding: 4px;
+      }
+
+      .multiline {
+        margin-top: 15px;
+        flex-direction: column;
+      }
+
+      .multiline .table-key {
+        margin-bottom: 10px;
+        border-bottom: 1px dashed #dddddd;
+        /* padding: 4px; */
+        width: unset;
+      }
+
+    </style>
+  </head>
+  <body>
+    <div class="table">
+      <div class="table-row">
+        <div class="table-key" title="Error ID">ERID: </div>
+        <div class="table-val">{{id}}</div>
+      </div>
+      {% if profile-id %}
+      <div class="table-row">
+        <div class="table-key" title="Profile ID">PFID: </div>
+        <div class="table-val">{{profile-id}}</div>
+      </div>
+      {% endif %}
+
+      {% if user-agent %}
+      <div class="table-row">
+        <div class="table-key">UAGT: </div>
+        <div class="table-val">{{user-agent}}</div>
+      </div>
+      {% endif %}
+
+      {% if frontend-version %}
+      <div class="table-row">
+        <div class="table-key">FVER: </div>
+        <div class="table-val">{{frontend-version}}</div>
+      </div>
+      {% endif %}
+
+      <div class="table-row">
+        <div class="table-key">BVER: </div>
+        <div class="table-val">{{version}}</div>
+      </div>
+
+      {% if host %}
+      <div class="table-row">
+        <div class="table-key">HOST: </div>
+        <div class="table-val">{{host}}</div>
+      </div>
+      {% endif %}
+
+      {% if tenant %}
+      <div class="table-row">
+        <div class="table-key">ENV: </div>
+        <div class="table-val">{{tenant}}</div>
+      </div>
+      {% endif %}
+
+      {% if public-uri %}
+      <div class="table-row">
+        <div class="table-key">PURI: </div>
+        <div class="table-val">{{public-uri}}</div>
+      </div>
+      {% endif %}
+
+      {% if type %}
+      <div class="table-row">
+        <div class="table-key">TYPE: </div>
+        <div class="table-val">{{type}}</div>
+      </div>
+      {% endif %}
+
+      {% if code %}
+      <div class="table-row">
+        <div class="table-key">CODE: </div>
+        <div class="table-val">{{code}}</div>
+      </div>
+      {% endif %}
+
+      {% if error %}
+      <div class="table-row">
+        <div class="table-key">CLSS: </div>
+        <div class="table-val">{{error.class}}</div>
+      </div>
+      {% endif %}
+
+      {% if error %}
+      <div class="table-row">
+        <div class="table-key">HINT: </div>
+        <div class="table-val">{{error.message}}</div>
+      </div>
+      {% endif %}
+
+      {% if method %}
+      <div class="table-row">
+        <div class="table-key">PATH: </div>
+        <div class="table-val">{{method|upper}} {{path}}</div>
+      </div>
+      {% endif %}
+
+      {% if explain %}
+        <div>(<a href="#explain">go to explain</a>)</div>
+      {% endif %}
+      {% if data %}
+        <div>(<a href="#edata">go to edata</a>)</div>
+      {% endif %}
+      {% if error %}
+        <div>(<a href="#trace">go to trace</a>)</div>
+      {% endif %}
+
+      {% if params %}
+      <div id="params" class="table-row multiline">
+        <div class="table-key">PARAMS: </div>
+        <div class="table-val">
+          <pre>{{params}}</pre>
+        </div>
+      </div>
+      {% endif %}
+
+      {% if explain %}
+      <div id="explain" class="table-row multiline">
+        <div class="table-key">EXPLAIN: </div>
+        <div class="table-val">
+          <pre>{{explain}}</pre>
+        </div>
+      </div>
+      {% endif %}
+
+      {% if data %}
+      <div id="edata" class="table-row multiline">
+        <div class="table-key">EDATA: </div>
+        <div class="table-val">
+          <pre>{{data}}</pre>
+        </div>
+      </div>
+      {% endif %}
+
+      {% if error %}
+      <div id="trace" class="table-row multiline">
+        <div class="table-key">TRACE:</div>
+        <div class="table-val">
+          <pre>{{error.trace}}</pre>
+        </div>
+      </div>
+      {% endif %}
+    </div>
+  </body>
+</html>
+

backend/resources/log4j2-bundle.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="info" monitorInterval="60">
-  <Appenders>
-    <Console name="console" target="SYSTEM_OUT">
-      <PatternLayout pattern="[%t] %level{length=1} %logger{36} - %msg%n"/>
-    </Console>
-  </Appenders>
-
-  <Loggers>
-    <Logger name="com.zaxxer.hikari" level="error" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="org.eclipse.jetty" level="info" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="app" level="debug" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Root level="info">
-      <AppenderRef ref="console"/>
-    </Root>
-  </Loggers>
-</Configuration>

backend/resources/log4j2-devenv.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="info" monitorInterval="30">
+  <Appenders>
+    <Console name="console" target="SYSTEM_OUT">
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
+    </Console>
+
+    <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
+      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
+      <Policies>
+        <SizeBasedTriggeringPolicy size="50M"/>
+      </Policies>
+      <DefaultRolloverStrategy max="9"/>
+    </RollingFile>
+
+    <JeroMQ name="zmq">
+      <Property name="endpoint">tcp://localhost:45556</Property>
+      <JsonLayout complete="false" compact="true" includeTimeMillis="true" stacktraceAsString="true" properties="true" />
+    </JeroMQ>
+  </Appenders>
+
+  <Loggers>
+    <Logger name="com.zaxxer.hikari" level="error"/>
+    <Logger name="io.lettuce" level="error" />
+    <Logger name="org.eclipse.jetty" level="error" />
+    <Logger name="org.postgresql" level="error" />
+
+    <Logger name="app.cli" level="debug" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+
+    <Logger name="app.loggers" level="debug" additivity="false">
+      <AppenderRef ref="main" level="debug" />
+    </Logger>
+
+    <Logger name="app" level="all" additivity="false">
+      <AppenderRef ref="main" level="trace" />
+      <AppenderRef ref="zmq" level="debug" />
+    </Logger>
+
+    <Logger name="penpot" level="debug" additivity="false">
+      <AppenderRef ref="main" level="debug" />
+      <AppenderRef ref="zmq" level="debug" />
+    </Logger>
+
+    <Logger name="user" level="trace" additivity="false">
+      <AppenderRef ref="main" level="trace" />
+    </Logger>
+
+    <Root level="info">
+      <AppenderRef ref="main" />
+    </Root>
+  </Loggers>
+</Configuration>

backend/resources/log4j2.xml

@@ -1,43 +1,25 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="info" monitorInterval="30">
+<Configuration status="info" monitorInterval="60">
   <Appenders>
     <Console name="console" target="SYSTEM_OUT">
       <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
     </Console>
-
-    <RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
-      <Policies>
-        <SizeBasedTriggeringPolicy size="50M"/>
-      </Policies>
-      <DefaultRolloverStrategy max="9"/>
-    </RollingFile>
-
-    <CljFn name="error-reporter" ns="app.error-reporter" fn="enqueue">
-      <PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
-    </CljFn>
   </Appenders>
 
   <Loggers>
-    <Logger name="com.zaxxer.hikari" level="error" additivity="false" />
-    <Logger name="org.eclipse.jetty" level="error" additivity="false" />
-    <Logger name="io.lettuce" level="error" additivity="false" />
-
-    <Logger name="app.cli" level="debug" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
-
-    <Logger name="app.error-reporter" level="debug" additivity="false">
-      <AppenderRef ref="console"/>
-    </Logger>
+    <Logger name="com.zaxxer.hikari" level="error" />
+    <Logger name="org.eclipse.jetty" level="error" />
 
     <Logger name="app" level="debug" additivity="false">
-      <AppenderRef ref="main" level="debug" />
-      <AppenderRef ref="error-reporter" level="error" />
+      <AppenderRef ref="console" />
+    </Logger>
+
+    <Logger name="penpot" level="fatal" additivity="false">
+      <AppenderRef ref="console" />
     </Logger>
 
     <Root level="info">
-      <AppenderRef ref="main" />
+      <AppenderRef ref="console" />
     </Root>
   </Loggers>
 </Configuration>

backend/scripts/build

@@ -0,0 +1,78 @@
+#!/usr/bin/env bb
+
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns build
+  (:require
+   [clojure.string :as str]
+   [clojure.java.io :as io]
+   [clojure.pprint :refer [pprint]]
+   [babashka.fs :as fs]
+   [babashka.process :refer [$ check]]))
+
+(defn split-cp
+  [data]
+  (str/split data #":"))
+
+(def classpath
+  (->> ($ clojure -Spath)
+       (check)
+       (:out)
+       (slurp)
+       (split-cp)
+       (map str/trim)))
+
+(def classpath-jars
+  (let [xfm (filter #(str/ends-with? % ".jar"))]
+    (into #{} xfm classpath)))
+
+(def classpath-paths
+  (let [xfm (comp (remove #(str/ends-with? % ".jar"))
+                  (filter #(.isDirectory (io/file %))))]
+    (into #{} xfm classpath)))
+
+(def version
+  (or (first *command-line-args*) "%version%"))
+
+;; Clean previous dist
+(-> ($ rm -rf "./target/dist") check)
+
+;; Create a new dist
+(-> ($ mkdir -p "./target/dist/deps") check)
+
+;; Copy all jar deps into dist
+(run! (fn [item] (-> ($ cp ~item "./target/dist/deps/") check)) classpath-jars)
+
+;; Create the application jar
+(spit "./target/dist/version.txt" version)
+(-> ($ jar cvf "./target/dist/deps/app.jar" -C ~(first classpath-paths) ".") check)
+(-> ($ jar uvf "./target/dist/deps/app.jar" -C "./target/dist" "version.txt") check)
+(run! (fn [item]
+        (-> ($ jar uvf "./target/dist/deps/app.jar" -C ~item ".")  check))
+      (rest classpath-paths))
+
+;; Copy logging configuration
+(-> ($ cp "./resources/log4j2.xml" "./target/dist/") check)
+
+;; Create classpath file
+(let [jars (->> (into ["app.jar"] classpath-jars)
+                (map fs/file-name)
+                (map #(fs/path "deps" %))
+                (map str))]
+  (spit "./target/dist/classpath" (str/join ":" jars)))
+
+;; Copy run script template
+(-> ($ cp "./scripts/run.template.sh" "./target/dist/run.sh") check)
+
+;; Copy run script template
+(-> ($ cp "./scripts/manage.template.sh" "./target/dist/manage.sh") check)
+
+;; Add exec permisions to scripts.
+(-> ($ chmod +x "./target/dist/run.sh") check)
+(-> ($ chmod +x "./target/dist/manage.sh") check)
+
+nil

backend/scripts/build.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-
-CLASSPATH=`(clojure -Spath)`
-NEWCP="./main:./common"
-
-rm -rf ./target/dist
-mkdir -p ./target/dist/deps
-
-for item in $(echo $CLASSPATH | tr ":" "\n"); do
-    if [ "${item: -4}" == ".jar" ]; then
-        cp $item ./target/dist/deps/;
-        BN="$(basename -- $item)"
-        NEWCP+=":./deps/$BN"
-    fi
-done
-
-cp ./resources/log4j2-bundle.xml ./target/dist/log4j2.xml
-cp -r ./src ./target/dist/main
-cp -r ./resources/emails ./target/dist/main/
-cp -r ../common ./target/dist/common
-
-echo $NEWCP > ./target/dist/classpath;
-
-tee -a ./target/dist/run.sh  >> /dev/null <<EOF
-#!/usr/bin/env bash
-
-CP="$NEWCP"
-
-# Exports
-
-# Find java executable
-set +e
-JAVA_CMD=\$(type -p java)
-
-set -e
-if [[ ! -n "\$JAVA_CMD" ]]; then
-  if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="\$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
-fi
-
-if [ -f ./environ ]; then
-   source ./environ
-fi
-
-set -x
-exec \$JAVA_CMD \$JVM_OPTS -Dapp.enable-asserts=false -classpath \$CP -Dlog4j.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main
-EOF
-
-chmod +x ./target/dist/run.sh

backend/scripts/import-generic-collections.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-clojure -Adev -m app.cli.collimp $@
-

backend/scripts/manage.template.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+exec $JAVA_CMD $JVM_OPTS -classpath $(cat classpath) -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "$@"

backend/scripts/prepare-release.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-if [ "$#" -e 0 ]; then
-    echo "Expecting parameters: 1=path to backend; 2=destination directory"
-    exit 1
-fi
-
-rm -rf $2 || exit 1;
-
-rsync -avr \
-      --exclude="/test" \
-      --exclude="/resources/public/media" \
-      --exclude="/target" \
-      --exclude="/scripts" \
-      --exclude="/.*" \
-      $1 $2;

backend/scripts/psql.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-PGPASSWORD=$APP_DATABASE_PASSWORD psql $APP_DATABASE_URI -U $APP_DATABASE_USERNAME

backend/scripts/repl

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
-set -ex
 
-# clojure -Ojmx-remote -A:dev -e "(set! *warn-on-reflection* true)" -m rebel-readline.main
-# clojure -Ojmx-remote -A:dev -J-XX:+UnlockExperimentalVMOptions -J-XX:+UseZGC -J-Xms128m -J-Xmx128m -m rebel-readline.main
-clojure -A:jmx-remote:dev -J-Xms128m -J-Xmx128m -M -m rebel-readline.main
+export PENPOT_ASSERTS_ENABLED=true
+
+export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Dlog4j2.configurationFile=log4j2-devenv.xml -J-XX:+UseZGC -J-XX:ConcGCThreads=1 -J-XX:-OmitStackTraceInFastThrow -J-Xms50m -J-Xmx512m";
+# export OPTIONS="$OPTIONS -J-XX:+UnlockDiagnosticVMOptions";
+# export OPTIONS="$OPTIONS -J-XX:-TieredCompilation -J-XX:CompileThreshold=10000";
+
+export OPTIONS_EVAL="nil"
+# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"
+
+set -ex
+exec clojure $OPTIONS -M -e "$OPTIONS_EVAL" -m rebel-readline.main

backend/scripts/run-tests-in-docker.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-clojure -Adev -m app.tests.main;

backend/scripts/run.template.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set +e
+JAVA_CMD=$(type -p java)
+
+set -e
+if [[ ! -n "$JAVA_CMD" ]]; then
+  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_CMD="$JAVA_HOME/bin/java"
+  else
+    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+    exit 1
+  fi
+fi
+
+if [ -f ./environ ]; then
+   source ./environ
+fi
+
+set -x
+exec $JAVA_CMD $JVM_OPTS -classpath "$(cat classpath)" -Dlog4j2.configurationFile=./log4j2.xml "$@" clojure.main -m app.main

backend/scripts/smtpd.sh

@@ -1,2 +0,0 @@
-#!/usr/bin/env bash
-python -m smtpd -n -c DebuggingServer localhost:25

backend/scripts/start-dev

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+export PENPOT_ASSERTS_ENABLED=true
+
 set -ex
 
 if [ ! -e ~/.fixtures-loaded ]; then
@@ -8,6 +10,6 @@ if [ ! -e ~/.fixtures-loaded ]; then
     touch ~/.fixtures-loaded
 fi
 
-clojure -M -m app.main
+clojure -A:dev -M -m app.main
 
 

backend/src/app/cli/fixtures.clj

@@ -2,24 +2,20 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.cli.fixtures
   "A initial fixtures."
   (:require
    [app.common.pages :as cp]
    [app.common.uuid :as uuid]
-   [app.config :as cfg]
    [app.db :as db]
-   [app.migrations]
-   [app.services.mutations.profile :as profile]
+   [app.main :as main]
+   [app.rpc.mutations.profile :as profile]
    [app.util.blob :as blob]
+   [app.util.logging :as l]
    [buddy.hashers :as hashers]
-   [clojure.tools.logging :as log]
-   [mount.core :as mount]))
+   [integrant.core :as ig]))
 
 (defn- mk-uuid
   [prefix & args]
@@ -71,18 +67,20 @@
        (#'profile/create-profile-relations conn)))
 
 (defn impl-run
-  [opts]
+  [pool opts]
   (let [rng (java.util.Random. 1)]
     (letfn [(create-profile [conn index]
               (let [id   (mk-uuid "profile" index)
-                    _    (log/info "create profile" id)
+                    _    (l/info :action "create profile"
+                                 :index index
+                                 :id id)
 
                     prof (register-profile conn
                                            {:id id
                                             :fullname (str "Profile " index)
                                             :password "123123"
-                                            :demo? true
-                                            :email (str "profile" index ".test@penpot.app")})
+                                            :is-demo true
+                                            :email (str "profile" index "@example.com")})
                     team-id  (:default-team-id prof)
                     owner-id id]
                 (let [project-ids (collect (partial create-project conn team-id owner-id)
@@ -91,29 +89,32 @@
                 prof))
 
             (create-profiles [conn]
-              (log/info "create profiles")
+              (l/info :action "create profiles")
               (collect (partial create-profile conn)
                        (range (:num-profiles opts))))
 
             (create-team [conn index]
               (let [id (mk-uuid "team" index)
                     name (str "Team" index)]
-                (log/info "create team" id)
+                (l/info :action "create team"
+                        :index index
+                        :id id)
                 (db/insert! conn :team {:id id
-                                        :name name
-                                        :photo ""})
+                                        :name name})
                 id))
 
             (create-teams [conn]
-              (log/info "create teams")
+              (l/info :action "create teams")
               (collect (partial create-team conn)
                        (range (:num-teams opts))))
 
             (create-file [conn owner-id project-id index]
               (let [id (mk-uuid "file" project-id index)
                     name (str "file" index)
-                    data (cp/make-file-data)]
-                (log/info "create file" id)
+                    data (cp/make-file-data id)]
+                (l/info :action "create file"
+                        :index index
+                        :id id)
                 (db/insert! conn :file
                             {:id id
                              :data (blob/encode data)
@@ -128,17 +129,25 @@
                 id))
 
             (create-files [conn owner-id project-id]
-              (log/info "create files")
+              (l/info :action "create files")
               (run! (partial create-file conn owner-id project-id)
                     (range (:num-files-per-project opts))))
 
             (create-project [conn team-id owner-id index]
-              (let [id (mk-uuid "project" team-id index)
-                    name (str "project " index)]
-                (log/info "create project" id)
+              (let [id        (if index
+                                (mk-uuid "project" team-id index)
+                                (mk-uuid "project" team-id))
+                    name      (if index
+                                (str "project " index)
+                                "Drafts")
+                    is-default (nil? index)]
+                (l/info :action "create project"
+                        :index index
+                        :id id)
                 (db/insert! conn :project
                             {:id id
                              :team-id team-id
+                             :is-default is-default
                              :name name})
                 (db/insert! conn :project-profile-rel
                             {:project-id id
@@ -149,10 +158,12 @@
                 id))
 
             (create-projects [conn team-id profile-ids]
-              (log/info "create projects")
+              (l/info :action "create projects")
               (let [owner-id (rng-nth rng profile-ids)
-                    project-ids (collect (partial create-project conn team-id owner-id)
-                                         (range (:num-projects-per-team opts)))]
+                    project-ids (conj
+                                  (collect (partial create-project conn team-id owner-id)
+                                         (range (:num-projects-per-team opts)))
+                                  (create-project conn team-id owner-id nil))]
                 (run! (partial create-files conn owner-id) project-ids)))
 
             (assign-profile-to-team [conn team-id owner? profile-id]
@@ -164,14 +175,16 @@
                            :can-edit true}))
 
             (setup-team [conn team-id profile-ids]
-              (log/info "setup team" team-id profile-ids)
+              (l/info :action "setup team"
+                      :team-id team-id
+                      :profile-ids (pr-str profile-ids))
               (assign-profile-to-team conn team-id true (first profile-ids))
               (run! (partial assign-profile-to-team conn team-id false)
                     (rest profile-ids))
               (create-projects conn team-id profile-ids))
 
             (assign-teams-and-profiles [conn teams profiles]
-              (log/info "assign teams and profiles")
+              (l/info :action "assign teams and profiles")
               (loop [team-id (first teams)
                      teams (rest teams)]
                 (when-not (nil? team-id)
@@ -186,9 +199,11 @@
                     id         (mk-uuid "file" "draft" owner-id index)
                     name       (str "file" index)
                     project-id (:default-project-id owner)
-                    data       (cp/make-file-data)]
+                    data       (cp/make-file-data id)]
 
-                (log/info "create draft file" id)
+                (l/info :action "create draft file"
+                        :index index
+                        :id id)
                 (db/insert! conn :file
                             {:id id
                              :data (blob/encode data)
@@ -206,33 +221,38 @@
               (run! (partial create-draft-file conn profile)
                     (range (:num-draft-files-per-profile opts))))
             ]
-      (db/with-atomic [conn db/pool]
+      (db/with-atomic [conn pool]
         (let [profiles (create-profiles conn)
               teams    (create-teams conn)]
           (assign-teams-and-profiles conn teams (map :id profiles))
           (run! (partial create-draft-files conn) profiles))))))
 
-(defn run*
-  [preset]
-  (let [preset (if (map? preset)
+(defn run-in-system
+  [system preset]
+  (let [pool   (:app.db/pool system)
+        preset (if (map? preset)
                  preset
                  (case preset
                    (nil "small" :small) preset-small
                    ;; "medium" preset-medium
                    ;; "big" preset-big
                    preset-small))]
-    (impl-run preset)))
+    (impl-run pool preset)))
 
 (defn run
-  [{:keys [preset]
-    :or {preset :small}}]
-  (try
-    (-> (mount/only #{#'app.config/config
-                      #'app.db/pool
-                      #'app.migrations/migrations})
-        (mount/start))
-    (run* preset)
-    (catch Exception e
-      (log/errorf e "Unhandled exception."))
-    (finally
-      (mount/stop))))
+  [{:keys [preset] :or {preset :small}}]
+  (let [config (select-keys main/system-config
+                            [:app.db/pool
+                             :app.telemetry/migrations
+                             :app.migrations/migrations
+                             :app.migrations/all
+                             :app.metrics/metrics])
+        _      (ig/load-namespaces config)
+        system (-> (ig/prep config)
+                   (ig/init))]
+    (try
+      (run-in-system system preset)
+      (catch Exception e
+        (l/error :hint "unhandled exception" :cause e))
+      (finally
+        (ig/halt! system)))))

backend/src/app/cli/manage.clj

@@ -0,0 +1,169 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.cli.manage
+  "A manage cli api."
+  (:require
+   [app.db :as db]
+   [app.main :as main]
+   [app.rpc.mutations.profile :as profile]
+   [app.rpc.queries.profile :refer [retrieve-profile-data-by-email]]
+   [app.util.logging :as l]
+   [clojure.string :as str]
+   [clojure.tools.cli :refer [parse-opts]]
+   [integrant.core :as ig])
+  (:import
+   java.io.Console))
+
+;; --- IMPL
+
+(defn init-system
+  []
+  (let [data (-> main/system-config
+                 (select-keys [:app.db/pool :app.metrics/metrics])
+                 (assoc :app.migrations/all {}))]
+    (-> data ig/prep ig/init)))
+
+(defn- read-from-console
+  [{:keys [label type] :or {type :text}}]
+  (let [^Console console (System/console)]
+    (when-not console
+      (l/error :hint "no console found, can proceed")
+      (System/exit 1))
+
+    (binding [*out* (.writer console)]
+      (print label " ")
+      (.flush *out*))
+
+    (case type
+      :text (.readLine console)
+      :password (String. (.readPassword console)))))
+
+(defn create-profile
+  [options]
+  (let [system   (init-system)
+        email    (or (:email options)
+                     (read-from-console {:label "Email:"}))
+        fullname (or (:fullname options)
+                     (read-from-console {:label "Full Name:"}))
+        password (or (:password options)
+                     (read-from-console {:label "Password:"
+                                         :type :password}))]
+    (try
+      (db/with-atomic [conn (:app.db/pool system)]
+        (->> (profile/create-profile conn
+                                     {:fullname fullname
+                                      :email email
+                                      :password password
+                                      :is-active true
+                                      :is-demo false})
+             (profile/create-profile-relations conn)))
+
+      (when (pos? (:verbosity options))
+        (println "User created successfully."))
+      (System/exit 0)
+
+      (catch Exception _e
+        (when (pos? (:verbosity options))
+          (println "Unable to create user, already exists."))
+        (System/exit 1)))))
+
+(defn reset-password
+  [options]
+  (let [system (init-system)]
+    (try
+      (db/with-atomic [conn (:app.db/pool system)]
+        (let [email    (or (:email options)
+                           (read-from-console {:label "Email:"}))
+              profile  (retrieve-profile-data-by-email conn email)]
+          (when-not profile
+            (when (pos? (:verbosity options))
+              (println "Profile does not exists."))
+            (System/exit 1))
+
+          (let [password (or (:password options)
+                             (read-from-console {:label "Password:"
+                                                 :type :password}))]
+            (profile/update-profile-password! conn (assoc profile :password password))
+            (when (pos? (:verbosity options))
+              (println "Password changed successfully.")))))
+      (System/exit 0)
+      (catch Exception e
+        (when (pos? (:verbosity options))
+          (println "Unable to change password."))
+        (when (= 2 (:verbosity options))
+          (.printStackTrace e))
+        (System/exit 1)))))
+
+;; --- CLI PARSE
+
+(def cli-options
+  ;; An option with a required argument
+  [["-u" "--email EMAIL" "Email Address"]
+   ["-p" "--password PASSWORD" "Password"]
+   ["-n" "--name FULLNAME" "Full Name"
+    :id :fullname]
+   ["-v" nil "Verbosity level"
+    :id :verbosity
+    :default 1
+    :update-fn inc]
+   ["-q" nil "Dont' print to console"
+    :id :verbosity
+    :update-fn (constantly 0)]
+   ["-h" "--help"]])
+
+(defn usage
+  [options-summary]
+  (->> ["Penpot CLI management."
+        ""
+        "Usage: manage [options] action"
+        ""
+        "Options:"
+        options-summary
+        ""
+        "Actions:"
+        "  create-profile    Create new profile."
+        "  reset-password    Reset profile password."
+        ""]
+       (str/join \newline)))
+
+(defn error-msg [errors]
+  (str "The following errors occurred while parsing your command:\n\n"
+       (str/join \newline errors)))
+
+(defn validate-args
+  "Validate command line arguments. Either return a map indicating the program
+  should exit (with a error message, and optional ok status), or a map
+  indicating the action the program should take and the options provided."
+  [args]
+  (let [{:keys [options arguments errors summary] :as opts} (parse-opts args cli-options)]
+    ;; (pp/pprint opts)
+    (cond
+      (:help options) ; help => exit OK with usage summary
+      {:exit-message (usage summary) :ok? true}
+
+      errors ; errors => exit with description of errors
+      {:exit-message (error-msg errors)}
+
+      ;; custom validation on arguments
+      :else
+      (let [action (first arguments)]
+        (if (#{"create-profile" "reset-password"} action)
+          {:action (first arguments) :options options}
+          {:exit-message (usage summary)})))))
+
+(defn exit [status msg]
+  (println msg)
+  (System/exit status))
+
+(defn -main
+  [& args]
+  (let [{:keys [action options exit-message ok?]} (validate-args args)]
+    (if exit-message
+      (exit (if ok? 0 1) exit-message)
+      (case action
+        "create-profile" (create-profile options)
+        "reset-password" (reset-password options)))))

backend/src/app/cli/media_loader.clj

@@ -1,232 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.cli.media-loader
-  "Media libraries importer (command line helper)."
-  #_(:require
-   [app.common.spec :as us]
-   [app.common.uuid :as uuid]
-   [app.config]
-   [app.db :as db]
-   [app.media-storage]
-   [app.media]
-   [app.migrations]
-   [app.services.mutations.files :as files]
-   [app.services.mutations.media :as media]
-   [app.services.mutations.projects :as projects]
-   [clojure.java.io :as io]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [datoteka.core :as fs]
-   [mount.core :as mount])
-  #_(:import
-   java.io.PushbackReader))
-
-;; --- Constants & Helpers
-
-;; (def ^:const +graphics-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6a")
-;; (def ^:const +colors-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6c")
-
-;; (s/def ::id ::us/uuid)
-;; (s/def ::name ::us/string)
-;; (s/def ::path ::us/string)
-;; (s/def ::regex #(instance? java.util.regex.Pattern %))
-
-;; (s/def ::import-graphics
-;;   (s/keys :req-un [::path ::regex]))
-
-;; (s/def ::import-color
-;;   (s/* (s/cat :name ::us/string :color ::us/color)))
-
-;; (s/def ::import-colors (s/coll-of ::import-color))
-
-;; (s/def ::import-library
-;;   (s/keys :req-un [::name]
-;;           :opt-un [::import-graphics ::import-colors]))
-
-;; (defn exit!
-;;   ([] (exit! 0))
-;;   ([code]
-;;    (System/exit code)))
-
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ;; Graphics Importer
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; (defn- create-media-object
-;;   [conn file-id media-object-id localpath]
-;;   (s/assert fs/path? localpath)
-;;   (s/assert ::us/uuid file-id)
-;;   (s/assert ::us/uuid media-object-id)
-;;   (let [filename (fs/name localpath)
-;;         extension (second (fs/split-ext filename))
-;;         file (io/as-file localpath)
-;;         mtype (case extension
-;;                 ".jpg"  "image/jpeg"
-;;                 ".png"  "image/png"
-;;                 ".webp" "image/webp"
-;;                 ".svg"  "image/svg+xml")]
-;;     (log/info "Creating image" filename media-object-id)
-;;     (media/create-media-object conn {:content {:tempfile localpath
-;;                                                :filename filename
-;;                                                :content-type mtype
-;;                                                :size (.length file)}
-;;                                      :id media-object-id
-;;                                      :file-id file-id
-;;                                      :name filename
-;;                                      :is-local false})))
-
-;; (defn- media-object-exists?
-;;   [conn id]
-;;   (s/assert ::us/uuid id)
-;;   (let [row (db/get-by-id conn :media-object id)]
-;;     (if row true false)))
-
-;; (defn- import-media-object-if-not-exists
-;;   [conn file-id fpath]
-;;   (s/assert ::us/uuid file-id)
-;;   (s/assert fs/path? fpath)
-;;   (let [media-object-id (uuid/namespaced +graphics-uuid-ns+ (str file-id (fs/name fpath)))]
-;;     (when-not (media-object-exists? conn media-object-id)
-;;       (create-media-object conn file-id media-object-id fpath))
-;;     media-object-id))
-
-;; (defn- import-graphics
-;;   [conn file-id {:keys [path regex]}]
-;;   (run! (fn [fpath]
-;;           (when (re-matches regex (str fpath))
-;;             (import-media-object-if-not-exists conn file-id fpath)))
-;;         (->> (fs/list-dir path)
-;;              (filter fs/regular-file?))))
-
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ;; Colors Importer
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; (defn- create-color
-;;   [conn file-id name content]
-;;   (s/assert ::us/uuid file-id)
-;;   (s/assert ::us/color content)
-;;   (let [color-id (uuid/namespaced +colors-uuid-ns+ (str file-id content))]
-;;     (log/info "Creating color" color-id "-" name content)
-;;     (colors/create-color conn {:id color-id
-;;                                :file-id file-id
-;;                                :name name
-;;                                :content content})
-;;     color-id))
-
-;; (defn- import-colors
-;;   [conn file-id colors]
-;;   (db/delete! conn :color {:file-id file-id})
-;;   (run! (fn [[name content]]
-;;           (create-color conn file-id name content))
-;;         (partition-all 2 colors)))
-
-
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ;; Library files Importer
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; (defn- library-file-exists?
-;;   [conn id]
-;;   (s/assert ::us/uuid id)
-;;   (let [row (db/get-by-id conn :file id)]
-;;     (if row true false)))
-
-;; (defn- create-library-file-if-not-exists
-;;   [conn project-id {:keys [name]}]
-;;   (let [id (uuid/namespaced +colors-uuid-ns+ name)]
-;;     (when-not (library-file-exists? conn id)
-;;       (log/info "Creating library-file:" name)
-;;       (files/create-file conn {:id id
-;;                                :profile-id uuid/zero
-;;                                :project-id project-id
-;;                                :name name
-;;                                :is-shared true})
-;;       (files/create-page conn {:file-id id}))
-;;     id))
-
-;; (defn- process-library
-;;   [conn basedir project-id {:keys [graphics colors] :as library}]
-;;   (us/verify ::import-library library)
-;;   (let [library-file-id (create-library-file-if-not-exists conn project-id library)]
-;;     (when graphics
-;;       (->> (assoc graphics :path (fs/join basedir (:path graphics)))
-;;            (import-graphics conn library-file-id)))
-;;     (when colors
-;;       (import-colors conn library-file-id colors))))
-
-
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ;; Entry Point
-;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; (defn- project-exists?
-;;   [conn id]
-;;   (s/assert ::us/uuid id)
-;;   (let [row (db/get-by-id conn :project id)]
-;;     (if row true false)))
-
-;; (defn- create-project-if-not-exists
-;;   [conn {:keys [name] :as project}]
-;;   (let [id (uuid/namespaced +colors-uuid-ns+ name)]
-;;     (when-not (project-exists? conn id)
-;;       (log/info "Creating project" name)
-;;       (projects/create-project conn {:id id
-;;                                      :team-id uuid/zero
-;;                                      :name name
-;;                                      :default? false}))
-;;     id))
-
-;; (defn- validate-path
-;;   [path]
-;;   (let [path (if (symbol? path) (str path) path)]
-;;     (log/infof "Trying to load config from '%s'." path)
-;;     (when-not path
-;;       (log/error "No path is provided")
-;;       (exit! -1))
-;;     (when-not (fs/exists? path)
-;;       (log/error "Path does not exists.")
-;;       (exit! -1))
-;;     (when (fs/directory? path)
-;;       (log/error "The provided path is a directory.")
-;;       (exit! -1))
-;;     (fs/path path)))
-
-;; (defn- read-file
-;;   [path]
-;;   (let [reader (PushbackReader. (io/reader path))]
-;;     [(fs/parent path)
-;;      (read reader)]))
-
-;; (defn run*
-;;   [path]
-;;   (let [[basedir libraries] (read-file path)]
-;;     (db/with-atomic [conn db/pool]
-;;       (let [project-id (create-project-if-not-exists conn {:name "System libraries"})]
-;;         (run! #(process-library conn basedir project-id %) libraries)))))
-
-;; (defn run
-;;   [{:keys [path] :as params}]
-;;   (log/infof "Starting media loader.")
-;;   (let [path (validate-path path)]
-
-;;     (try
-;;       (-> (mount/only #{#'app.config/config
-;;                         #'app.db/pool
-;;                         #'app.migrations/migrations
-;;                         #'app.media/semaphore
-;;                         #'app.media-storage/media-storage})
-;;           (mount/start))
-;;       (run* path)
-;;       (catch Exception e
-;;         (log/errorf e "Unhandled exception."))
-;;       (finally
-;;         (mount/stop)))))
-

backend/src/app/cli/migrate_media.clj

@@ -0,0 +1,129 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.cli.migrate-media
+  (:require
+   [app.common.media :as cm]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.main :as main]
+   [app.storage :as sto]
+   [app.util.logging :as l]
+   [cuerdas.core :as str]
+   [datoteka.core :as fs]
+   [integrant.core :as ig]))
+
+(declare migrate-profiles)
+(declare migrate-teams)
+(declare migrate-file-media)
+
+(defn run-in-system
+  [system]
+  (db/with-atomic [conn (:app.db/pool system)]
+    (let [system (assoc system ::conn conn)]
+      (migrate-profiles system)
+      (migrate-teams system)
+      (migrate-file-media system))
+    system))
+
+(defn run
+  []
+  (let [config (select-keys main/system-config
+                            [:app.db/pool
+                             :app.migrations/migrations
+                             :app.metrics/metrics
+                             :app.storage.s3/backend
+                             :app.storage.db/backend
+                             :app.storage.fs/backend
+                             :app.storage/storage])]
+    (ig/load-namespaces config)
+    (try
+      (-> (ig/prep config)
+          (ig/init)
+          (run-in-system)
+          (ig/halt!))
+      (catch Exception e
+        (l/error :hint "unhandled exception" :cause e)))))
+
+
+;; --- IMPL
+
+(defn migrate-profiles
+  [{:keys [::conn] :as system}]
+  (letfn [(retrieve-profiles [conn]
+            (->> (db/exec! conn ["select * from profile"])
+                 (filter #(not (str/empty? (:photo %))))
+                 (seq)))]
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
+          storage (-> (:app.storage/storage system)
+                      (assoc :conn conn))]
+      (doseq [profile (retrieve-profiles conn)]
+        (let [path  (fs/path (:photo profile))
+              full  (-> (fs/join base path)
+                        (fs/normalize))
+              ext   (fs/ext path)
+              mtype (cm/format->mtype (keyword ext))
+              obj   (sto/put-object storage {:content (sto/content full)
+                                             :content-type mtype})]
+          (db/update! conn :profile
+                      {:photo-id (:id obj)}
+                      {:id (:id profile)}))))))
+
+(defn migrate-teams
+  [{:keys [::conn] :as system}]
+  (letfn [(retrieve-teams [conn]
+            (->> (db/exec! conn ["select * from team"])
+                 (filter #(not (str/empty? (:photo %))))
+                 (seq)))]
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
+          storage (-> (:app.storage/storage system)
+                      (assoc :conn conn))]
+      (doseq [team (retrieve-teams conn)]
+        (let [path  (fs/path (:photo team))
+              full  (-> (fs/join base path)
+                        (fs/normalize))
+              ext   (fs/ext path)
+              mtype (cm/format->mtype (keyword ext))
+              obj   (sto/put-object storage {:content (sto/content full)
+                                             :content-type mtype})]
+          (db/update! conn :team
+                      {:photo-id (:id obj)}
+                      {:id (:id team)}))))))
+
+
+
+(defn migrate-file-media
+  [{:keys [::conn] :as system}]
+  (letfn [(retrieve-media-objects [conn]
+            (->> (db/exec! conn ["select fmo.id, fmo.path, fth.path as thumbnail_path
+                                    from file_media_object as fmo
+                                    join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"])
+                 (seq)))]
+    (let [base    (fs/path (cf/get :storage-fs-old-directory))
+          storage (-> (:app.storage/storage system)
+                      (assoc :conn conn))]
+      (doseq [mobj (retrieve-media-objects conn)]
+        (let [img-path  (fs/path (:path mobj))
+              thm-path  (fs/path (:thumbnail-path mobj))
+              img-path  (-> (fs/join base img-path)
+                            (fs/normalize))
+              thm-path  (-> (fs/join base thm-path)
+                            (fs/normalize))
+              img-ext   (fs/ext img-path)
+              thm-ext   (fs/ext thm-path)
+
+              img-mtype (cm/format->mtype (keyword img-ext))
+              thm-mtype (cm/format->mtype (keyword thm-ext))
+
+              img-obj   (sto/put-object storage {:content (sto/content img-path)
+                                                 :content-type img-mtype})
+              thm-obj   (sto/put-object storage {:content (sto/content thm-path)
+                                                 :content-type thm-mtype})]
+
+          (db/update! conn :file-media-object
+                      {:media-id (:id img-obj)
+                       :thumbnail-id (:id thm-obj)}
+                      {:id (:id mobj)}))))))

backend/src/app/config.clj

@@ -2,220 +2,314 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.config
   "A configuration management."
+  (:refer-clojure :exclude [get])
   (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
    [app.common.spec :as us]
    [app.common.version :as v]
    [app.util.time :as dt]
+   [clojure.core :as c]
+   [clojure.java.io :as io]
+   [clojure.pprint :as pprint]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
    [environ.core :refer [env]]
-   [mount.core :refer [defstate]]))
+   [integrant.core :as ig]))
+
+(prefer-method print-method
+               clojure.lang.IRecord
+               clojure.lang.IDeref)
+
+(prefer-method pprint/simple-dispatch
+               clojure.lang.IPersistentMap
+               clojure.lang.IDeref)
+
+(defmethod ig/init-key :default
+  [_ data]
+  (d/without-nils data))
+
+(defmethod ig/prep-key :default
+  [_ data]
+  (if (map? data)
+    (d/without-nils data)
+    data))
 
 (def defaults
   {:http-server-port 6060
-   :http-server-cors "http://localhost:3449"
-   :database-uri "postgresql://127.0.0.1/penpot"
+   :host "devenv"
+   :tenant "dev"
+   :database-uri "postgresql://postgres/penpot"
    :database-username "penpot"
    :database-password "penpot"
-   :secret-key "default"
 
-   :media-directory "resources/public/media"
-   :assets-directory "resources/public/static"
+   :default-blob-version 3
+   :loggers-zmq-uri "tcp://localhost:45556"
 
-   :public-uri "http://localhost:3449/"
-   :redis-uri "redis://localhost/0"
-   :media-uri "http://localhost:3449/media/"
-   :assets-uri "http://localhost:3449/static/"
+   :asserts-enabled false
 
-   :image-process-max-threads 2
+   :public-uri "http://localhost:3449"
+   :redis-uri "redis://redis/0"
+
+   :srepl-host "127.0.0.1"
+   :srepl-port 6062
+
+   :storage-backend :fs
+
+   :storage-fs-directory "assets"
+   :storage-s3-region :eu-central-1
+   :storage-s3-bucket "penpot-devenv-assets-pre"
+
+   :feedback-destination "info@example.com"
+   :feedback-enabled false
+
+   :assets-path "/internal/assets/"
+
+   :rlimits-password 10
+   :rlimits-image 2
 
    :smtp-enabled false
-   :smtp-default-reply-to "no-reply@example.com"
-   :smtp-default-from "no-reply@example.com"
+   :smtp-default-reply-to "Penpot <no-reply@example.com>"
+   :smtp-default-from "Penpot <no-reply@example.com>"
 
-   :host "devenv"
+   :profile-complaint-max-age (dt/duration {:days 7})
+   :profile-complaint-threshold 2
+
+   :profile-bounce-max-age (dt/duration {:days 7})
+   :profile-bounce-threshold 10
 
    :allow-demo-users true
    :registration-enabled true
-   :registration-domain-whitelist ""
-   :debug-humanize-transit true
 
-   ;; This is the time should transcurr after the last page
-   ;; modification in order to make the file ellegible for
-   ;; trimming. The value only supports s(econds) m(inutes) and
-   ;; h(ours) as time unit.
-   :file-trimming-threshold "72h"
+   :telemetry-enabled false
+   :telemetry-uri "https://telemetry.penpot.app/"
 
-   ;; LDAP auth disabled by default. Set ldap-auth-host to enable
-   ;:ldap-auth-host "ldap.mysupercompany.com"
-   ;:ldap-auth-port 389
-   ;:ldap-bind-dn "cn=admin,dc=ldap,dc=mysupercompany,dc=com"
-   ;:ldap-bind-password "verysecure"
-   ;:ldap-auth-ssl false
-   ;:ldap-auth-starttls false
-   ;:ldap-auth-base-dn "ou=People,dc=ldap,dc=mysupercompany,dc=com"
+   :ldap-user-query "(|(uid=:username)(mail=:username))"
+   :ldap-attrs-username "uid"
+   :ldap-attrs-email "mail"
+   :ldap-attrs-fullname "cn"
+   :ldap-attrs-photo "jpegPhoto"
 
-   :ldap-auth-user-query "(|(uid=$username)(mail=$username))"
-   :ldap-auth-username-attribute "uid"
-   :ldap-auth-email-attribute "mail"
-   :ldap-auth-fullname-attribute "displayName"
-   :ldap-auth-avatar-attribute "jpegPhoto"})
+   ;; a server prop key where initial project is stored.
+   :initial-project-skey "initial-project"
+   })
 
-(s/def ::http-server-port ::us/integer)
-(s/def ::http-server-debug ::us/boolean)
-(s/def ::http-server-cors ::us/string)
-(s/def ::database-username (s/nilable ::us/string))
+(s/def ::audit-enabled ::us/boolean)
+(s/def ::audit-archive-enabled ::us/boolean)
+(s/def ::audit-archive-uri ::us/string)
+(s/def ::audit-archive-gc-enabled ::us/boolean)
+(s/def ::audit-archive-gc-max-age ::dt/duration)
+
+(s/def ::secret-key ::us/string)
+(s/def ::allow-demo-users ::us/boolean)
+(s/def ::asserts-enabled ::us/boolean)
+(s/def ::assets-path ::us/string)
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
-(s/def ::redis-uri ::us/string)
-(s/def ::assets-uri ::us/string)
-(s/def ::assets-directory ::us/string)
-(s/def ::media-uri ::us/string)
-(s/def ::media-directory ::us/string)
-(s/def ::secret-key ::us/string)
-
-(s/def ::host ::us/string)
+(s/def ::database-username (s/nilable ::us/string))
+(s/def ::default-blob-version ::us/integer)
 (s/def ::error-report-webhook ::us/string)
-(s/def ::smtp-enabled ::us/boolean)
-(s/def ::smtp-default-reply-to ::us/email)
-(s/def ::smtp-default-from ::us/email)
-(s/def ::smtp-host ::us/string)
-(s/def ::smtp-port ::us/integer)
-(s/def ::smtp-username (s/nilable ::us/string))
-(s/def ::smtp-password (s/nilable ::us/string))
-(s/def ::smtp-tls ::us/boolean)
-(s/def ::smtp-ssl ::us/boolean)
-(s/def ::allow-demo-users ::us/boolean)
-(s/def ::registration-enabled ::us/boolean)
-(s/def ::registration-domain-whitelist ::us/string)
-(s/def ::debug-humanize-transit ::us/boolean)
-(s/def ::public-uri ::us/string)
-(s/def ::backend-uri ::us/string)
-
-(s/def ::image-process-max-threads ::us/integer)
-(s/def ::file-trimming-threshold ::dt/duration)
-
-(s/def ::google-client-id ::us/string)
-(s/def ::google-client-secret ::us/string)
-
+(s/def ::feedback-destination ::us/string)
+(s/def ::feedback-enabled ::us/boolean)
+(s/def ::feedback-token ::us/string)
+(s/def ::github-client-id ::us/string)
+(s/def ::github-client-secret ::us/string)
+(s/def ::gitlab-base-uri ::us/string)
 (s/def ::gitlab-client-id ::us/string)
 (s/def ::gitlab-client-secret ::us/string)
-(s/def ::gitlab-base-uri ::us/string)
-
-(s/def ::ldap-auth-host ::us/string)
-(s/def ::ldap-auth-port ::us/integer)
+(s/def ::google-client-id ::us/string)
+(s/def ::google-client-secret ::us/string)
+(s/def ::oidc-client-id ::us/string)
+(s/def ::oidc-client-secret ::us/string)
+(s/def ::oidc-base-uri ::us/string)
+(s/def ::oidc-token-uri ::us/string)
+(s/def ::oidc-auth-uri ::us/string)
+(s/def ::oidc-user-uri ::us/string)
+(s/def ::oidc-scopes ::us/set-of-str)
+(s/def ::oidc-roles ::us/set-of-str)
+(s/def ::oidc-roles-attr ::us/keyword)
+(s/def ::host ::us/string)
+(s/def ::http-server-port ::us/integer)
+(s/def ::http-session-idle-max-age ::dt/duration)
+(s/def ::http-session-updater-batch-max-age ::dt/duration)
+(s/def ::http-session-updater-batch-max-size ::us/integer)
+(s/def ::initial-project-skey ::us/string)
+(s/def ::ldap-attrs-email ::us/string)
+(s/def ::ldap-attrs-fullname ::us/string)
+(s/def ::ldap-attrs-photo ::us/string)
+(s/def ::ldap-attrs-username ::us/string)
+(s/def ::ldap-base-dn ::us/string)
 (s/def ::ldap-bind-dn ::us/string)
 (s/def ::ldap-bind-password ::us/string)
-(s/def ::ldap-auth-ssl ::us/boolean)
-(s/def ::ldap-auth-starttls ::us/boolean)
-(s/def ::ldap-auth-base-dn ::us/string)
-(s/def ::ldap-auth-user-query ::us/string)
-(s/def ::ldap-auth-username-attribute ::us/string)
-(s/def ::ldap-auth-email-attribute ::us/string)
-(s/def ::ldap-auth-fullname-attribute ::us/string)
-(s/def ::ldap-auth-avatar-attribute ::us/string)
+(s/def ::ldap-host ::us/string)
+(s/def ::ldap-port ::us/integer)
+(s/def ::ldap-ssl ::us/boolean)
+(s/def ::ldap-starttls ::us/boolean)
+(s/def ::ldap-user-query ::us/string)
+(s/def ::loggers-loki-uri ::us/string)
+(s/def ::loggers-zmq-uri ::us/string)
+(s/def ::media-directory ::us/string)
+(s/def ::media-uri ::us/string)
+(s/def ::profile-bounce-max-age ::dt/duration)
+(s/def ::profile-bounce-threshold ::us/integer)
+(s/def ::profile-complaint-max-age ::dt/duration)
+(s/def ::profile-complaint-threshold ::us/integer)
+(s/def ::public-uri ::us/string)
+(s/def ::redis-uri ::us/string)
+(s/def ::registration-domain-whitelist ::us/set-of-str)
+(s/def ::registration-enabled ::us/boolean)
+(s/def ::rlimits-image ::us/integer)
+(s/def ::rlimits-password ::us/integer)
+(s/def ::smtp-default-from ::us/string)
+(s/def ::smtp-default-reply-to ::us/string)
+(s/def ::smtp-enabled ::us/boolean)
+(s/def ::smtp-host ::us/string)
+(s/def ::smtp-password (s/nilable ::us/string))
+(s/def ::smtp-port ::us/integer)
+(s/def ::smtp-ssl ::us/boolean)
+(s/def ::smtp-tls ::us/boolean)
+(s/def ::smtp-username (s/nilable ::us/string))
+(s/def ::srepl-host ::us/string)
+(s/def ::srepl-port ::us/integer)
+(s/def ::storage-backend ::us/keyword)
+(s/def ::storage-fs-directory ::us/string)
+(s/def ::storage-s3-bucket ::us/string)
+(s/def ::storage-s3-region ::us/keyword)
+(s/def ::telemetry-enabled ::us/boolean)
+(s/def ::telemetry-uri ::us/string)
+(s/def ::telemetry-with-taiga ::us/boolean)
+(s/def ::tenant ::us/string)
 
 (s/def ::config
-  (s/keys :opt-un [::http-server-cors
-                   ::http-server-debug
-                   ::http-server-port
-                   ::google-client-id
-                   ::google-client-secret
-                   ::gitlab-client-id
-                   ::gitlab-client-secret
-                   ::gitlab-base-uri
-                   ::redis-uri
-                   ::public-uri
-                   ::database-username
+  (s/keys :opt-un [::secret-key
+                   ::allow-demo-users
+                   ::audit-enabled
+                   ::audit-archive-enabled
+                   ::audit-archive-uri
+                   ::audit-archive-gc-enabled
+                   ::audit-archive-gc-max-age
+                   ::asserts-enabled
                    ::database-password
                    ::database-uri
-                   ::assets-directory
-                   ::assets-uri
-                   ::media-directory
-                   ::media-uri
+                   ::database-username
+                   ::default-blob-version
                    ::error-report-webhook
-                   ::secret-key
+                   ::feedback-destination
+                   ::feedback-enabled
+                   ::feedback-token
+                   ::github-client-id
+                   ::github-client-secret
+                   ::gitlab-base-uri
+                   ::gitlab-client-id
+                   ::gitlab-client-secret
+                   ::google-client-id
+                   ::google-client-secret
+                   ::oidc-client-id
+                   ::oidc-client-secret
+                   ::oidc-base-uri
+                   ::oidc-token-uri
+                   ::oidc-auth-uri
+                   ::oidc-user-uri
+                   ::oidc-scopes
+                   ::oidc-roles-attr
+                   ::oidc-roles
+                   ::host
+                   ::http-server-port
+                   ::http-session-idle-max-age
+                   ::http-session-updater-batch-max-age
+                   ::http-session-updater-batch-max-size
+                   ::initial-project-skey
+                   ::ldap-attrs-email
+                   ::ldap-attrs-fullname
+                   ::ldap-attrs-photo
+                   ::ldap-attrs-username
+                   ::ldap-base-dn
+                   ::ldap-bind-dn
+                   ::ldap-bind-password
+                   ::ldap-host
+                   ::ldap-port
+                   ::ldap-ssl
+                   ::ldap-starttls
+                   ::ldap-user-query
+                   ::local-assets-uri
+                   ::loggers-loki-uri
+                   ::loggers-zmq-uri
+                   ::profile-bounce-max-age
+                   ::profile-bounce-threshold
+                   ::profile-complaint-max-age
+                   ::profile-complaint-threshold
+                   ::public-uri
+                   ::redis-uri
+                   ::registration-domain-whitelist
+                   ::registration-enabled
+                   ::rlimits-image
+                   ::rlimits-password
                    ::smtp-default-from
                    ::smtp-default-reply-to
                    ::smtp-enabled
                    ::smtp-host
-                   ::smtp-port
-                   ::smtp-username
                    ::smtp-password
-                   ::smtp-tls
+                   ::smtp-port
                    ::smtp-ssl
-                   ::host
-                   ::file-trimming-threshold
-                   ::debug-humanize-transit
-                   ::allow-demo-users
-                   ::registration-enabled
-                   ::registration-domain-whitelist
-                   ::image-process-max-threads
-                   ::ldap-auth-host
-                   ::ldap-auth-port
-                   ::ldap-bind-dn
-                   ::ldap-bind-password
-                   ::ldap-auth-ssl
-                   ::ldap-auth-starttls
-                   ::ldap-auth-base-dn
-                   ::ldap-auth-user-query
-                   ::ldap-auth-username-attribute
-                   ::ldap-auth-email-attribute
-                   ::ldap-auth-fullname-attribute
-                   ::ldap-auth-avatar-attribute]))
+                   ::smtp-tls
+                   ::smtp-username
+                   ::srepl-host
+                   ::srepl-port
+                   ::storage-backend
+                   ::storage-fs-directory
+                   ::storage-s3-bucket
+                   ::storage-s3-region
+                   ::telemetry-enabled
+                   ::telemetry-uri
+                   ::telemetry-referer
+                   ::telemetry-with-taiga
+                   ::tenant]))
 
-(defn env->config
-  [env]
-  (reduce-kv
-   (fn [acc k v]
-     (cond-> acc
-       (str/starts-with? (name k) "penpot-")
-       (assoc (keyword (subs (name k) 7)) v)
+(defn read-env
+  [prefix]
+  (let [prefix (str prefix "-")
+        len    (count prefix)]
+    (reduce-kv
+     (fn [acc k v]
+       (cond-> acc
+         (str/starts-with? (name k) prefix)
+         (assoc (keyword (subs (name k) len)) v)))
+     {}
+     env)))
 
-       (str/starts-with? (name k) "app-")
-       (assoc (keyword (subs (name k) 4)) v)))
-   {}
-   env))
+(defn- read-config
+  []
+  (try
+    (->> (read-env "penpot")
+         (merge defaults)
+         (us/conform ::config))
+    (catch Throwable e
+      (when (ex/ex-info? e)
+        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")
+        (println "Error on validating configuration:")
+        (println (:explain (ex-data e))
+        (println ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;")))
+      (throw e))))
 
-(defn read-config
-  [env]
-  (->> (env->config env)
-       (merge defaults)
-       (us/conform ::config)))
+(def version (v/parse (or (some-> (io/resource "version.txt")
+                                  (slurp)
+                                  (str/trim))
+                          "%version%")))
+(def config (atom (read-config)))
 
-(defn read-test-config
-  [env]
-  (assoc (read-config env)
-         :redis-uri "redis://redis/1"
-         :database-uri "postgresql://postgres/penpot_test"
-         :media-directory "/tmp/app/media"
-         :assets-directory "/tmp/app/static"
-         :migrations-verbose false))
+(def deletion-delay
+  (dt/duration {:days 7}))
 
-(defstate config
-  :start (read-config env))
-
-(def default-deletion-delay
-  (dt/duration {:hours 48}))
-
-(def version
-  (delay (v/parse "%version%")))
-
-(defn smtp
-  [cfg]
-  {:host (:smtp-host cfg "localhost")
-   :port (:smtp-port cfg 25)
-   :default-reply-to (:smtp-default-reply-to cfg)
-   :default-from (:smtp-default-from cfg)
-   :tls (:smtp-tls cfg)
-   :enabled (:smtp-enabled cfg)
-   :username (:smtp-username cfg)
-   :password (:smtp-password cfg)})
+(defn get
+  "A configuration getter. Helps code be more testable."
+  ([key]
+   (c/get @config key))
+  ([key default]
+   (c/get @config key default)))
 
+;; Set value for all new threads bindings.
+(alter-var-root #'*assert* (constantly (get :asserts-enabled)))

backend/src/app/db.clj

@@ -2,63 +2,116 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.db
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.geom.point :as gpt]
-   [app.config :as cfg]
+   [app.common.spec :as us]
+   [app.db.sql :as sql]
    [app.metrics :as mtx]
+   [app.util.json :as json]
+   [app.util.logging :as l]
+   [app.util.migrations :as mg]
    [app.util.time :as dt]
    [app.util.transit :as t]
-   [clojure.data.json :as json]
+   [clojure.java.io :as io]
    [clojure.spec.alpha :as s]
-   [clojure.string :as str]
-   [mount.core :as mount :refer [defstate]]
+   [integrant.core :as ig]
    [next.jdbc :as jdbc]
-   [next.jdbc.date-time :as jdbc-dt]
-   [next.jdbc.optional :as jdbc-opt]
-   [next.jdbc.sql :as jdbc-sql]
-   [next.jdbc.sql.builder :as jdbc-bld])
+   [next.jdbc.date-time :as jdbc-dt])
   (:import
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
+   java.lang.AutoCloseable
    java.sql.Connection
    java.sql.Savepoint
+   org.postgresql.PGConnection
    org.postgresql.geometric.PGpoint
+   org.postgresql.largeobject.LargeObject
+   org.postgresql.largeobject.LargeObjectManager
    org.postgresql.jdbc.PgArray
    org.postgresql.util.PGInterval
    org.postgresql.util.PGobject))
 
+(declare open)
+(declare create-pool)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Initialization
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare instrument-jdbc!)
+
+(s/def ::name keyword?)
+(s/def ::uri ::us/not-empty-string)
+(s/def ::min-pool-size ::us/integer)
+(s/def ::max-pool-size ::us/integer)
+(s/def ::migrations map?)
+
+(defmethod ig/pre-init-spec ::pool [_]
+  (s/keys :req-un [::uri ::name ::min-pool-size ::max-pool-size ::migrations ::mtx/metrics]))
+
+(defmethod ig/init-key ::pool
+  [_ {:keys [migrations metrics] :as cfg}]
+  (l/info :action "initialize connection pool"
+          :name (d/name (:name cfg))
+          :uri (:uri cfg))
+  (instrument-jdbc! (:registry metrics))
+  (let [pool (create-pool cfg)]
+    (when (seq migrations)
+      (with-open [conn ^AutoCloseable (open pool)]
+        (mg/setup! conn)
+        (doseq [[name steps] migrations]
+          (mg/migrate! conn {:name (d/name name) :steps steps}))))
+    pool))
+
+(defmethod ig/halt-key! ::pool
+  [_ pool]
+  (.close ^HikariDataSource pool))
+
+(defn- instrument-jdbc!
+  [registry]
+  (mtx/instrument-vars!
+   [#'next.jdbc/execute-one!
+    #'next.jdbc/execute!]
+   {:registry registry
+    :type :counter
+    :name "database_query_total"
+    :help "An absolute counter of database queries."}))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; API & Impl
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (def initsql
-  (str "SET statement_timeout = 10000;\n"
-       "SET idle_in_transaction_session_timeout = 30000;"))
+  (str "SET statement_timeout = 120000;\n"
+       "SET idle_in_transaction_session_timeout = 120000;"))
 
 (defn- create-datasource-config
-  [cfg]
-  (let [dburi    (:database-uri cfg)
-        username (:database-username cfg)
-        password (:database-password cfg)
-        config (HikariConfig.)
-        mfactory (PrometheusMetricsTrackerFactory. mtx/registry)]
+  [{:keys [metrics] :as cfg}]
+  (let [dburi    (:uri cfg)
+        username (:username cfg)
+        password (:password cfg)
+        config   (HikariConfig.)
+        mtf      (PrometheusMetricsTrackerFactory. (:registry metrics))]
     (doto config
       (.setJdbcUrl (str "jdbc:" dburi))
-      (.setPoolName "main")
+      (.setPoolName (d/name (:name cfg)))
       (.setAutoCommit true)
       (.setReadOnly false)
       (.setConnectionTimeout 8000)  ;; 8seg
-      (.setValidationTimeout 4000)  ;; 4seg
-      (.setIdleTimeout 300000)      ;; 5min
-      (.setMaxLifetime 900000)      ;; 15min
-      (.setMinimumIdle 0)
-      (.setMaximumPoolSize 15)
+      (.setValidationTimeout 8000)  ;; 8seg
+      (.setIdleTimeout 120000)      ;; 2min
+      (.setMaxLifetime 1800000)     ;; 30min
+      (.setMinimumIdle (:min-pool-size cfg 0))
+      (.setMaximumPoolSize (:max-pool-size cfg 30))
+      (.setMetricsTrackerFactory mtf)
       (.setConnectionInitSql initsql)
-      (.setMetricsTrackerFactory mfactory))
+      (.setInitializationFailTimeout -1))
     (when username (.setUsername config username))
     (when password (.setPassword config password))
     config))
@@ -71,7 +124,7 @@
 
 (defn pool-closed?
   [pool]
-  (.isClosed ^com.zaxxer.hikari.HikariDataSource pool))
+  (.isClosed ^HikariDataSource pool))
 
 (defn- create-pool
   [cfg]
@@ -79,66 +132,104 @@
     (jdbc-dt/read-as-instant)
     (HikariDataSource. dsc)))
 
-(declare pool)
+(defn unwrap
+  [conn klass]
+  (.unwrap ^Connection conn klass))
 
-(defstate pool
-  :start (create-pool cfg/config)
-  :stop (.close pool))
+(defn lobj-manager
+  [conn]
+  (let [conn (unwrap conn org.postgresql.PGConnection)]
+    (.getLargeObjectAPI ^PGConnection conn)))
+
+(defn lobj-create
+  [manager]
+  (.createLO ^LargeObjectManager manager LargeObjectManager/READWRITE))
+
+(defn lobj-open
+  ([manager oid]
+   (lobj-open manager oid {}))
+  ([manager oid {:keys [mode] :or {mode :rw}}]
+   (let [mode (case mode
+                (:r :read) LargeObjectManager/READ
+                (:w :write) LargeObjectManager/WRITE
+                (:rw :read+write) LargeObjectManager/READWRITE)]
+     (.open ^LargeObjectManager manager (long oid) mode))))
+
+(defn lobj-unlink
+  [manager oid]
+  (.unlink ^LargeObjectManager manager (long oid)))
+
+(extend-type LargeObject
+  io/IOFactory
+  (make-reader [lobj opts]
+    (let [^InputStream is (.getInputStream ^LargeObject lobj)]
+      (io/make-reader is opts)))
+  (make-writer [lobj opts]
+    (let [^OutputStream os (.getOutputStream ^LargeObject lobj)]
+      (io/make-writer os opts)))
+  (make-input-stream [lobj opts]
+    (let [^InputStream is (.getInputStream ^LargeObject lobj)]
+      (io/make-input-stream is opts)))
+  (make-output-stream [lobj opts]
+    (let [^OutputStream os (.getOutputStream ^LargeObject lobj)]
+      (io/make-output-stream os opts))))
 
 (defmacro with-atomic
   [& args]
   `(jdbc/with-transaction ~@args))
 
-(defn- kebab-case [s] (str/replace s #"_" "-"))
-(defn- snake-case [s] (str/replace s #"-" "_"))
-(defn- as-kebab-maps
-  [rs opts]
-  (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
-
-(defn open
-  []
+(defn ^Connection open
+  [pool]
   (jdbc/get-connection pool))
 
 (defn exec!
   ([ds sv]
    (exec! ds sv {}))
   ([ds sv opts]
-   (jdbc/execute! ds sv (assoc opts :builder-fn as-kebab-maps))))
+   (jdbc/execute! ds sv (assoc opts :builder-fn sql/as-kebab-maps))))
 
 (defn exec-one!
   ([ds sv] (exec-one! ds sv {}))
   ([ds sv opts]
-   (jdbc/execute-one! ds sv (assoc opts :builder-fn as-kebab-maps))))
-
-(def ^:private default-options
-  {:table-fn snake-case
-   :column-fn snake-case
-   :builder-fn as-kebab-maps})
+   (jdbc/execute-one! ds sv (assoc opts :builder-fn sql/as-kebab-maps))))
 
 (defn insert!
-  [ds table params]
-  (jdbc-sql/insert! ds table params default-options))
+  ([ds table params] (insert! ds table params nil))
+  ([ds table params opts]
+   (exec-one! ds
+              (sql/insert table params opts)
+              (assoc opts :return-keys true))))
+
+(defn insert-multi!
+  ([ds table cols rows] (insert-multi! ds table cols rows nil))
+  ([ds table cols rows opts]
+   (exec! ds
+          (sql/insert-multi table cols rows opts)
+          (assoc opts :return-keys true))))
 
 (defn update!
-  [ds table params where]
-  (let [opts (assoc default-options :return-keys true)]
-    (jdbc-sql/update! ds table params where opts)))
+  ([ds table params where] (update! ds table params where nil))
+  ([ds table params where opts]
+   (exec-one! ds
+              (sql/update table params where opts)
+              (assoc opts :return-keys true))))
 
 (defn delete!
-  [ds table params]
-  (let [opts (assoc default-options :return-keys true)]
-    (jdbc-sql/delete! ds table params opts)))
+  ([ds table params] (delete! ds table params nil))
+  ([ds table params opts]
+   (exec-one! ds
+              (sql/delete table params opts)
+              (assoc opts :return-keys true))))
 
 (defn get-by-params
   ([ds table params]
    (get-by-params ds table params nil))
-  ([ds table params opts]
-   (let [opts (cond-> (merge default-options opts)
-                (:for-update opts)
-                (assoc :suffix "for update"))
-         res  (exec-one! ds (jdbc-bld/for-query table params opts) opts)]
-     (when (or (:deleted-at res) (not res))
-       (ex/raise :type :not-found))
+  ([ds table params {:keys [uncheked] :or {uncheked false} :as opts}]
+   (let [res (exec-one! ds (sql/select table params opts))]
+     (when (and (not uncheked)
+                (or (:deleted-at res) (not res)))
+       (ex/raise :type :not-found
+                 :hint "database object not found"))
      res)))
 
 (defn get-by-id
@@ -151,10 +242,7 @@
   ([ds table params]
    (query ds table params nil))
   ([ds table params opts]
-   (let [opts (cond-> (merge default-options opts)
-                (:for-update opts)
-                (assoc :suffix "for update"))]
-     (exec! ds (jdbc-bld/for-query table params opts) opts))))
+   (exec! ds (sql/select table params opts))))
 
 (defn pgobject?
   [v]
@@ -180,6 +268,14 @@
   [p]
   (PGpoint. (:x p) (:y p)))
 
+(defn create-array
+  [conn type objects]
+  (let [^PGConnection conn (unwrap conn org.postgresql.PGConnection)]
+    (if (coll? objects)
+      (.createArrayOf conn ^String type (into-array Object objects))
+      (.createArrayOf conn ^String type objects))))
+
+
 (defn decode-pgpoint
   [^PGpoint v]
   (gpt/point (.-x v) (.-y v)))
@@ -212,7 +308,7 @@
     (pginterval data)
 
     (dt/duration? data)
-    (->> (/ (.toMillis data) 1000.0)
+    (->> (/ (.toMillis ^java.time.Duration data) 1000.0)
          (format "%s seconds")
          (pginterval))
 
@@ -225,7 +321,7 @@
         val (.getValue o)]
     (if (or (= typ "json")
             (= typ "jsonb"))
-      (json/read-str val :key-fn keyword)
+      (json/decode-str val)
       val)))
 
 (defn decode-transit-pgobject
@@ -237,6 +333,12 @@
       (t/decode-str val)
       val)))
 
+(defn inet
+  [ip-addr]
+  (doto (org.postgresql.util.PGobject.)
+    (.setType "inet")
+    (.setValue (str ip-addr))))
+
 (defn tjson
   "Encode as transit json."
   [data]
@@ -249,7 +351,7 @@
   [data]
   (doto (org.postgresql.util.PGobject.)
     (.setType "jsonb")
-    (.setValue (json/write-str data))))
+    (.setValue (json/encode-str data))))
 
 (defn pgarray->set
   [v]
@@ -258,11 +360,3 @@
 (defn pgarray->vector
   [v]
   (vec (.getArray ^PgArray v)))
-
-;; Instrumentation
-
-(mtx/instrument-with-counter!
- {:var [#'jdbc/execute-one!
-        #'jdbc/execute!]
-  :id "database__query_counter"
-  :help "An absolute counter of database queries."})

backend/src/app/db/sql.clj

@@ -0,0 +1,62 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.db.sql
+  (:refer-clojure :exclude [update])
+  (:require
+   [clojure.string :as str]
+   [next.jdbc.optional :as jdbc-opt]
+   [next.jdbc.sql.builder :as sql]))
+
+(defn kebab-case [s] (str/replace s #"_" "-"))
+(defn snake-case [s] (str/replace s #"-" "_"))
+
+(def default-opts
+  {:table-fn snake-case
+   :column-fn snake-case})
+
+(defn as-kebab-maps
+  [rs opts]
+  (jdbc-opt/as-unqualified-modified-maps rs (assoc opts :label-fn kebab-case)))
+
+(defn insert
+  ([table key-map]
+   (insert table key-map nil))
+  ([table key-map opts]
+   (let [opts (merge default-opts opts)
+         opts (cond-> opts
+                (:on-conflict-do-nothing opts)
+                (assoc :suffix "ON CONFLICT DO NOTHING"))]
+     (sql/for-insert table key-map opts))))
+
+(defn insert-multi
+  [table cols rows opts]
+  (let [opts (merge default-opts opts)]
+    (sql/for-insert-multi table cols rows opts)))
+
+(defn select
+  ([table where-params]
+   (select table where-params nil))
+  ([table where-params opts]
+   (let [opts (merge default-opts opts)
+         opts (cond-> opts
+                (:for-update opts)
+                (assoc :suffix "FOR UPDATE"))]
+     (sql/for-query table where-params opts))))
+
+(defn update
+  ([table key-map where-params]
+   (update table key-map where-params nil))
+  ([table key-map where-params opts]
+   (let [opts (merge default-opts opts)]
+     (sql/for-update table key-map where-params opts))))
+
+(defn delete
+  ([table where-params]
+   (delete table where-params nil))
+  ([table where-params opts]
+   (let [opts (merge default-opts opts)]
+     (sql/for-delete table where-params opts))))

backend/src/app/emails.clj

@@ -2,28 +2,22 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.emails
   "Main api for send emails."
   (:require
    [app.common.spec :as us]
    [app.config :as cfg]
-   [app.tasks :as tasks]
+   [app.db :as db]
+   [app.db.sql :as sql]
    [app.util.emails :as emails]
-   [clojure.spec.alpha :as s]))
+   [app.util.logging :as l]
+   [app.worker :as wrk]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
 
-;; --- Defaults
-
-(defn default-context
-  []
-  {:assets-uri (:assets-uri cfg/config)
-   :public-uri (:public-uri cfg/config)})
-
-;; --- Public API
+;; --- PUBLIC API
 
 (defn render
   [email-factory context]
@@ -31,17 +25,78 @@
 
 (defn send!
   "Schedule the email for sending."
-  [conn email-factory context]
-  (us/verify fn? email-factory)
-  (us/verify map? context)
-  (let [email (email-factory context)]
-    (tasks/submit! conn {:name "sendmail"
-                         :delay 0
-                         :max-retries 1
-                         :priority 200
-                         :props email})))
+  [{:keys [::conn ::factory] :as context}]
+  (us/verify fn? factory)
+  (us/verify some? conn)
+  (let [email (factory context)]
+    (wrk/submit! (assoc email
+                        ::wrk/task :sendmail
+                        ::wrk/delay 0
+                        ::wrk/max-retries 1
+                        ::wrk/priority 200
+                        ::wrk/conn conn))))
 
-;; --- Emails
+
+;; --- BOUNCE/COMPLAINS HANDLING
+
+(def sql:profile-complaint-report
+  "select (select count(*)
+             from profile_complaint_report
+            where type = 'complaint'
+              and profile_id = ?
+              and created_at > now() - ?::interval) as complaints,
+          (select count(*)
+             from profile_complaint_report
+            where type = 'bounce'
+              and profile_id = ?
+              and created_at > now() - ?::interval) as bounces;")
+
+(defn allow-send-emails?
+  [conn profile]
+  (when-not (:is-muted profile false)
+    (let [complaint-threshold (cfg/get :profile-complaint-threshold)
+          complaint-max-age   (cfg/get :profile-complaint-max-age)
+          bounce-threshold    (cfg/get :profile-bounce-threshold)
+          bounce-max-age      (cfg/get :profile-bounce-max-age)
+
+          {:keys [complaints bounces] :as result}
+          (db/exec-one! conn [sql:profile-complaint-report
+                              (:id profile)
+                              (db/interval complaint-max-age)
+                              (:id profile)
+                              (db/interval bounce-max-age)])]
+
+      (and (< complaints complaint-threshold)
+           (< bounces bounce-threshold)))))
+
+(defn has-complaint-reports?
+  ([conn email] (has-complaint-reports? conn email nil))
+  ([conn email {:keys [threshold] :or {threshold 1}}]
+   (let [reports (db/exec! conn (sql/select :global-complaint-report
+                                            {:email email :type "complaint"}
+                                            {:limit 10}))]
+     (>= (count reports) threshold))))
+
+(defn has-bounce-reports?
+  ([conn email] (has-bounce-reports? conn email nil))
+  ([conn email {:keys [threshold] :or {threshold 1}}]
+   (let [reports (db/exec! conn (sql/select :global-complaint-report
+                                            {:email email :type "bounce"}
+                                            {:limit 10}))]
+     (>= (count reports) threshold))))
+
+
+;; --- EMAIL FACTORIES
+
+(s/def ::subject ::us/string)
+(s/def ::content ::us/string)
+
+(s/def ::feedback
+  (s/keys :req-un [::subject ::content]))
+
+(def feedback
+  "A profile feedback email."
+  (emails/template-factory ::feedback))
 
 (s/def ::name ::us/string)
 (s/def ::register
@@ -49,7 +104,7 @@
 
 (def register
   "A new profile registration welcome email."
-  (emails/template-factory ::register default-context))
+  (emails/template-factory ::register))
 
 (s/def ::token ::us/string)
 (s/def ::password-recovery
@@ -57,7 +112,7 @@
 
 (def password-recovery
   "A password recovery notification email."
-  (emails/template-factory ::password-recovery default-context))
+  (emails/template-factory ::password-recovery))
 
 (s/def ::pending-email ::us/email)
 (s/def ::change-email
@@ -65,7 +120,7 @@
 
 (def change-email
   "Password change confirmation email"
-  (emails/template-factory ::change-email default-context))
+  (emails/template-factory ::change-email))
 
 (s/def :internal.emails.invite-to-team/invited-by ::us/string)
 (s/def :internal.emails.invite-to-team/team ::us/string)
@@ -78,4 +133,50 @@
 
 (def invite-to-team
   "Teams member invitation email."
-  (emails/template-factory ::invite-to-team default-context))
+  (emails/template-factory ::invite-to-team))
+
+
+;; --- SENDMAIL TASK
+
+(declare send-console!)
+
+(s/def ::username ::cfg/smtp-username)
+(s/def ::password ::cfg/smtp-password)
+(s/def ::tls ::cfg/smtp-tls)
+(s/def ::ssl ::cfg/smtp-ssl)
+(s/def ::host ::cfg/smtp-host)
+(s/def ::port ::cfg/smtp-port)
+(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
+(s/def ::default-from ::cfg/smtp-default-from)
+(s/def ::enabled ::cfg/smtp-enabled)
+
+(defmethod ig/pre-init-spec ::sendmail-handler [_]
+  (s/keys :req-un [::enabled]
+          :opt-un [::username
+                   ::password
+                   ::tls
+                   ::ssl
+                   ::host
+                   ::port
+                   ::default-from
+                   ::default-reply-to]))
+
+(defmethod ig/init-key ::sendmail-handler
+  [_ cfg]
+  (fn [{:keys [props] :as task}]
+    (if (:enabled cfg)
+      (emails/send! cfg props)
+      (send-console! cfg props))))
+
+(defn- send-console!
+  [cfg email]
+  (let [baos (java.io.ByteArrayOutputStream.)
+        mesg (emails/smtp-message cfg email)]
+    (.writeTo mesg baos)
+    (let [out (with-out-str
+                (println "email console dump:")
+                (println "******** start email" (:id email) "**********")
+                (println (.toString baos))
+                (println "******** end email "(:id email) "**********"))]
+      (l/info :email out))))
+

backend/src/app/error_reporter.clj

@@ -1,83 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
-
-(ns app.error-reporter
-  "A mattermost integration for error reporting."
-  (:require
-   [app.common.exceptions :as ex]
-   [app.common.spec :as us]
-   [app.config :as cfg]
-   [app.db :as db]
-   [app.tasks :as tasks]
-   [app.util.async :as aa]
-   [app.worker :as wrk]
-   [app.util.http :as http]
-   [clojure.core.async :as a]
-   [clojure.data.json :as json]
-   [clojure.spec.alpha :as s]
-   [clojure.tools.logging :as log]
-   [cuerdas.core :as str]
-   [mount.core :as mount :refer [defstate]]
-   [promesa.exec :as px]))
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Public API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defonce enqueue identity)
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Implementation
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(defn- send-to-mattermost!
-  [log-event]
-  (try
-    (let [text  (str/fmt "Unhandled exception: `host='%s'`, `version=%s`.\n@channel ⇊\n```%s\n```"
-                         (:host cfg/config)
-                         (:full @cfg/version)
-                         (str log-event))
-          rsp   (http/send! {:uri (:error-reporter-webhook cfg/config)
-                             :method :post
-                             :headers {"content-type" "application/json"}
-                             :body (json/write-str {:text text})})]
-      (when (not= (:status rsp) 200)
-        (log/warnf "Error reporting webhook replying with unexpected status: %s\n%s"
-                   (:status rsp)
-                   (pr-str rsp))))
-    (catch Exception e
-      (log/warnf e "Unexpected exception on error reporter."))))
-
-(defn- send!
-  [val]
-  (aa/thread-call wrk/executor (partial send-to-mattermost! val)))
-
-(defn- start
-  []
-  (let [qch (a/chan (a/sliding-buffer 128))]
-    (log/info "Starting error reporter loop.")
-
-    ;; Only enable when a valid URL is provided.
-    (when (:error-reporter-webhook cfg/config)
-      (alter-var-root #'enqueue (constantly #(a/>!! qch %)))
-      (a/go-loop []
-        (let [val (a/<! qch)]
-          (if (nil? val)
-            (do
-              (log/info "Closing error reporting loop.")
-              (alter-var-root #'enqueue (constantly identity)))
-            (do
-              (a/<! (send! val))
-              (recur))))))
-
-    qch))
-
-(defstate reporter
-  :start (start)
-  :stop  (a/close! reporter))

backend/src/app/http.clj

@@ -2,79 +2,156 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http
   (:require
-   [app.config :as cfg]
-   [app.http.auth :as auth]
-   [app.http.auth.gitlab :as gitlab]
-   [app.http.auth.google :as google]
-   [app.http.auth.ldap :as ldap]
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
    [app.http.errors :as errors]
-   [app.http.handlers :as handlers]
    [app.http.middleware :as middleware]
-   [app.http.session :as session]
-   [app.http.ws :as ws]
    [app.metrics :as mtx]
-   [clojure.tools.logging :as log]
-   [mount.core :as mount :refer [defstate]]
-   [reitit.ring :as rring]
-   [ring.adapter.jetty9 :as jetty]))
+   [app.util.logging :as l]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [reitit.ring :as rr]
+   [ring.adapter.jetty9 :as jetty])
+  (:import
+   org.eclipse.jetty.server.Server
+   org.eclipse.jetty.server.handler.ErrorHandler
+   org.eclipse.jetty.server.handler.StatisticsHandler))
 
-(defn- create-router
-  []
-  (rring/router
-   [["/metrics" {:get mtx/dump}]
-    ["/api" {:middleware [[middleware/format-response-body]
-                          [middleware/parse-request-body]
-                          [middleware/errors errors/handle]
+(declare router-handler)
+
+(s/def ::handler fn?)
+(s/def ::router some?)
+(s/def ::ws (s/map-of ::us/string fn?))
+(s/def ::port ::us/integer)
+(s/def ::name ::us/string)
+
+(defmethod ig/pre-init-spec ::server [_]
+  (s/keys :req-un [::port]
+          :opt-un [::ws ::name ::mtx/metrics ::router ::handler]))
+
+(defmethod ig/prep-key ::server
+  [_ cfg]
+  (merge {:name "http"} (d/without-nils cfg)))
+
+(defmethod ig/init-key ::server
+  [_ {:keys [handler router ws port name metrics] :as opts}]
+  (l/info :msg "starting http server" :port port :name name)
+  (let [pre-start (fn [^Server server]
+                    (let [handler (doto (ErrorHandler.)
+                                    (.setShowStacks true)
+                                    (.setServer server))]
+                      (.setErrorHandler server ^ErrorHandler handler)
+                      (when metrics
+                        (let [stats (StatisticsHandler.)]
+                          (.setHandler ^StatisticsHandler stats (.getHandler server))
+                          (.setHandler server stats)
+                          (mtx/instrument-jetty! (:registry metrics) stats)))))
+
+        options   (merge
+                   {:port port
+                    :h2c? true
+                    :join? false
+                    :allow-null-path-info true
+                    :configurator pre-start}
+                   (when (seq ws)
+                     {:websockets ws}))
+
+        handler   (cond
+                    (fn? handler)  handler
+                    (some? router) (router-handler router)
+                    :else (ex/raise :type :internal
+                                    :code :invalid-argument
+                                    :hint "Missing `handler` or `router` option."))
+
+        server    (jetty/run-jetty handler options)]
+    (assoc opts :server server)))
+
+(defmethod ig/halt-key! ::server
+  [_ {:keys [server name port] :as opts}]
+  (l/info :msg "stoping http server"
+          :name name
+          :port port)
+  (jetty/stop-server server))
+
+(defn- router-handler
+  [router]
+  (let [handler (rr/ring-handler router
+                                 (rr/routes
+                                  (rr/create-resource-handler {:path "/"})
+                                  (rr/create-default-handler))
+                                 {:middleware [middleware/server-timing]})]
+    (fn [request]
+      (try
+        (handler request)
+        (catch Throwable e
+          (try
+            (let [cdata (errors/get-error-context request e)]
+              (l/update-thread-context! cdata)
+              (l/error :hint "unhandled exception"
+                       :message (ex-message e)
+                       :error-id (str (:id cdata))
+                       :cause e))
+            {:status 500 :body "internal server error"}
+            (catch Throwable e
+              (l/error :hint "unhandled exception"
+                       :message (ex-message e)
+                       :cause e)
+              {:status 500 :body "internal server error"})))))))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Http Main Handler (Router)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(s/def ::rpc map?)
+(s/def ::session map?)
+(s/def ::oauth map?)
+(s/def ::storage map?)
+(s/def ::assets map?)
+(s/def ::feedback fn?)
+
+(defmethod ig/pre-init-spec ::router [_]
+  (s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
+
+(defmethod ig/init-key ::router
+  [_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
+  (rr/router
+   [["/metrics" {:get (:handler metrics)}]
+    ["/assets" {:middleware [[middleware/format-response-body]
+                             [middleware/errors errors/handle]
+                             [middleware/cookies]
+                             (:middleware session)]}
+     ["/by-id/:id" {:get (:objects-handler assets)}]
+     ["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
+     ["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
+
+    ["/dbg"
+     ["/error-by-id/:id" {:get (:error-report-handler cfg)}]]
+
+    ["/webhooks"
+     ["/sns" {:post (:sns-webhook cfg)}]]
+
+    ["/api" {:middleware [[middleware/etag]
+                          [middleware/format-response-body]
                           [middleware/params]
                           [middleware/multipart-params]
                           [middleware/keyword-params]
+                          [middleware/parse-request-body]
+                          [middleware/errors errors/handle]
                           [middleware/cookies]]}
 
-     ["/oauth"
-      ["/google" {:post google/auth}]
-      ["/google/callback" {:get google/callback}]
-      ["/gitlab" {:post gitlab/auth}]
-      ["/gitlab/callback" {:get gitlab/callback}]]
+     ["/feedback" {:middleware [(:middleware session)]
+                   :post feedback}]
 
-     ["/echo" {:get handlers/echo-handler
-               :post handlers/echo-handler}]
+     ["/auth/oauth/:provider" {:post (:handler oauth)}]
+     ["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
 
-     ["/login" {:handler auth/login-handler
-                :method :post}]
-     ["/logout" {:handler auth/logout-handler
-                 :method :post}]
-     ["/login-ldap" {:handler ldap/auth
-                     :method :post}]
-
-     ["/w" {:middleware [session/middleware]}
-      ["/query/:type" {:get handlers/query-handler}]
-      ["/mutation/:type" {:post handlers/mutation-handler}]]]]))
-
-(defn start-server
-  []
-  (let [wsockets {"/ws/notifications" ws/handler}
-        options  {:port (:http-server-port cfg/config)
-                  :h2c? true
-                  :join? false
-                  :allow-null-path-info true
-                  :websockets wsockets}
-        handler  (rring/ring-handler
-                  (create-router)
-                  (constantly {:status 404, :body ""})
-                  {:middleware [[middleware/development-resources]
-                                [middleware/development-cors]
-                                [middleware/metrics]]})]
-    (log/infof "Http server listening on http://localhost:%s/"
-               (:http-server-port cfg/config))
-    (jetty/run-jetty handler options)))
-
-(defstate server
-  :start (start-server)
-  :stop (.stop server))
+     ["/rpc" {:middleware [(:middleware session)]}
+      ["/query/:type" {:get (:query-handler rpc)
+                       :post (:query-handler rpc)}]
+      ["/mutation/:type" {:post (:mutation-handler rpc)}]]]]))

backend/src/app/http/assets.clj

@@ -0,0 +1,110 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.assets
+  "Assets related handlers."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uri :as u]
+   [app.db :as db]
+   [app.metrics :as mtx]
+   [app.storage :as sto]
+   [app.util.time :as dt]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(def ^:private cache-max-age
+  (dt/duration {:hours 24}))
+
+(def ^:private signature-max-age
+  (dt/duration {:hours 24 :minutes 15}))
+
+(defn coerce-id
+  [id]
+  (let [res (us/uuid-conformer id)]
+    (when-not (uuid? res)
+      (ex/raise :type :not-found
+                :hint "object not found"))
+    res))
+
+(defn- get-file-media-object
+  [{:keys [pool] :as storage} id]
+  (let [id   (coerce-id id)
+        mobj (db/exec-one! pool ["select * from file_media_object where id=?" id])]
+    (when-not mobj
+      (ex/raise :type :not-found
+                :hint "object does not found"))
+      mobj))
+
+(defn- serve-object
+  [{:keys [storage] :as cfg} obj]
+  (let [mdata   (meta obj)
+        backend (sto/resolve-backend storage (:backend obj))]
+    (case (:type backend)
+      :db
+      {:status 200
+       :headers {"content-type" (:content-type mdata)
+                 "cache-control" (str "max-age=" (inst-ms cache-max-age))}
+       :body (sto/get-object-data storage obj)}
+
+      :s3
+      (let [url (sto/get-object-url storage obj {:max-age signature-max-age})]
+        {:status 307
+         :headers {"location" (str url)
+                   "x-host"   (:host url)
+                   "cache-control" (str "max-age=" (inst-ms cache-max-age))}
+         :body ""})
+
+      :fs
+      (let [purl (u/uri (:assets-path cfg))
+            purl (u/join purl (sto/object->relative-path obj))]
+        {:status 204
+         :headers {"x-accel-redirect" (:path purl)
+                   "content-type" (:content-type mdata)
+                   "cache-control" (str "max-age=" (inst-ms cache-max-age))}
+         :body ""}))))
+
+(defn- generic-handler
+  [{:keys [storage] :as cfg} _request id]
+  (let [obj (sto/get-object storage id)]
+    (if obj
+      (serve-object cfg obj)
+      {:status 404 :body ""})))
+
+(defn objects-handler
+  [cfg request]
+  (let [id (get-in request [:path-params :id])]
+    (generic-handler cfg request (coerce-id id))))
+
+(defn file-objects-handler
+  [{:keys [storage] :as cfg} request]
+  (let [id   (get-in request [:path-params :id])
+        mobj (get-file-media-object storage id)]
+    (generic-handler cfg request (:media-id mobj))))
+
+(defn file-thumbnails-handler
+  [{:keys [storage] :as cfg} request]
+  (let [id   (get-in request [:path-params :id])
+        mobj (get-file-media-object storage id)]
+    (generic-handler cfg request (or (:thumbnail-id mobj) (:media-id mobj)))))
+
+
+;; --- Initialization
+
+(s/def ::storage some?)
+(s/def ::assets-path ::us/string)
+(s/def ::cache-max-age ::dt/duration)
+(s/def ::signature-max-age ::dt/duration)
+
+(defmethod ig/pre-init-spec ::handlers [_]
+  (s/keys :req-un [::storage ::mtx/metrics ::assets-path ::cache-max-age ::signature-max-age]))
+
+(defmethod ig/init-key ::handlers
+  [_ cfg]
+  {:objects-handler #(objects-handler cfg %)
+   :file-objects-handler #(file-objects-handler cfg %)
+   :file-thumbnails-handler #(file-thumbnails-handler cfg %)})

backend/src/app/http/auth.clj

@@ -1,31 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth
-  (:require
-   [app.http.session :as session]
-   [app.services.mutations :as sm]))
-
-(defn login-handler
-  [req]
-  (let [data    (:body-params req)
-        uagent  (get-in req [:headers "user-agent"])
-        profile (sm/handle (assoc data ::sm/type :login))
-        id      (session/create (:id profile) uagent)]
-    {:status 200
-     :cookies (session/cookies id)
-     :body profile}))
-
-(defn logout-handler
-  [req]
-  (some-> (session/extract-auth-token req)
-          (session/delete))
-  {:status 200
-   :cookies (session/cookies "" {:max-age -1})
-   :body ""})

backend/src/app/http/auth/gitlab.clj

@@ -1,147 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.gitlab
-  (:require
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
-   [app.http.session :as session]
-   [app.services.mutations :as sm]
-   [app.services.tokens :as tokens]
-   [app.util.http :as http]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.tools.logging :as log]
-   [lambdaisland.uri :as uri]))
-
-(def default-base-gitlab-uri "https://gitlab.com")
-
-(def scope "read_user")
-
-(defn- build-redirect-url
-  []
-  (let [public (uri/uri (:public-uri cfg/config))]
-    (str (assoc public :path "/api/oauth/gitlab/callback"))))
-
-
-(defn- build-oauth-uri
-  []
-  (let [base-uri (uri/uri (:gitlab-base-uri cfg/config default-base-gitlab-uri))]
-    (assoc base-uri :path "/oauth/authorize")))
-
-
-(defn- build-token-url
-  []
-  (let [base-uri (uri/uri (:gitlab-base-uri cfg/config default-base-gitlab-uri))]
-    (str (assoc base-uri :path "/oauth/token"))))
-
-
-(defn- build-user-info-url
-  []
-  (let [base-uri (uri/uri (:gitlab-base-uri cfg/config default-base-gitlab-uri))]
-    (str (assoc base-uri :path "/api/v4/user"))))
-
-
-(defn- get-access-token
-  [code]
-  (let [params {:client_id (:gitlab-client-id cfg/config)
-                :client_secret (:gitlab-client-secret cfg/config)
-                :code code
-                :grant_type "authorization_code"
-                :redirect_uri (build-redirect-url)}
-        req    {:method :post
-                :headers {"content-type" "application/x-www-form-urlencoded"}
-                :uri (build-token-url)
-                :body (uri/map->query-string params)}
-        res    (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-gitlab
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        (get data "access_token"))
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from gitlab access token request" e)
-        nil))))
-
-
-(defn- get-user-info
-  [token]
-  (let [req {:uri (build-user-info-url)
-             :headers {"Authorization" (str "Bearer " token)}
-             :method :get}
-        res (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-gitlab
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        ;; (clojure.pprint/pprint data)
-        {:email (get data "email")
-         :fullname (get data "name")})
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from gitlab access token request" e)
-        nil))))
-
-(defn auth
-  [_req]
-  (let [token  (tokens/generate
-                {:iss :gitlab-oauth
-                 :exp (dt/in-future "15m")})
-
-        params {:client_id (:gitlab-client-id cfg/config)
-                :redirect_uri (build-redirect-url)
-                :response_type "code"
-                :state token
-                :scope scope}
-        query  (uri/map->query-string params)
-        uri    (-> (build-oauth-uri)
-                   (assoc :query query))]
-    {:status 200
-     :body {:redirect-uri (str uri)}}))
-
-(defn callback
-  [req]
-  (let [token (get-in req [:params :state])
-        _     (tokens/verify token {:iss :gitlab-oauth})
-        info  (some-> (get-in req [:params :code])
-                      (get-access-token)
-                      (get-user-info))]
-
-    (when-not info
-      (ex/raise :type :authentication
-                :code :unable-to-authenticate-with-gitlab))
-
-    (let [profile (sm/handle {::sm/type :login-or-register
-                              :email (:email info)
-                              :fullname (:fullname info)})
-          uagent  (get-in req [:headers "user-agent"])
-
-          token   (tokens/generate
-                   {:iss :auth
-                    :exp (dt/in-future "15m")
-                    :profile-id (:id profile)})
-
-          uri     (-> (uri/uri (:public-uri cfg/config))
-                      (assoc :path "/#/auth/verify-token")
-                      (assoc :query (uri/map->query-string {:token token})))
-          sid     (session/create (:id profile) uagent)]
-
-      {:status 302
-       :headers {"location" (str uri)}
-       :cookies (session/cookies sid)
-       :body ""})))

backend/src/app/http/auth/google.clj

@@ -1,131 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.google
-  (:require
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
-   [app.http.session :as session]
-   [app.services.mutations :as sm]
-   [app.services.tokens :as tokens]
-   [app.util.http :as http]
-   [app.util.time :as dt]
-   [clojure.data.json :as json]
-   [clojure.tools.logging :as log]
-   [lambdaisland.uri :as uri]))
-
-(def base-goauth-uri "https://accounts.google.com/o/oauth2/v2/auth")
-
-(def scope
-  (str "email profile "
-       "https://www.googleapis.com/auth/userinfo.email "
-       "https://www.googleapis.com/auth/userinfo.profile "
-       "openid"))
-
-(defn- build-redirect-url
-  []
-  (let [public (uri/uri (:public-uri cfg/config))]
-    (str (assoc public :path "/api/oauth/google/callback"))))
-
-(defn- get-access-token
-  [code]
-  (let [params {:code code
-                :client_id (:google-client-id cfg/config)
-                :client_secret (:google-client-secret cfg/config)
-                :redirect_uri (build-redirect-url)
-                :grant_type "authorization_code"}
-        req    {:method :post
-                :headers {"content-type" "application/x-www-form-urlencoded"}
-                :uri "https://oauth2.googleapis.com/token"
-                :body (uri/map->query-string params)}
-        res    (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-google
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        (get data "access_token"))
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from google access token request" e)
-        nil))))
-
-
-(defn- get-user-info
-  [token]
-  (let [req {:uri "https://openidconnect.googleapis.com/v1/userinfo"
-             :headers {"Authorization" (str "Bearer " token)}
-             :method :get}
-        res (http/send! req)]
-
-    (when (not= 200 (:status res))
-      (ex/raise :type :internal
-                :code :invalid-response-from-google
-                :context {:status (:status res)
-                          :body (:body res)}))
-
-    (try
-      (let [data (json/read-str (:body res))]
-        ;; (clojure.pprint/pprint data)
-        {:email (get data "email")
-         :fullname (get data "name")})
-      (catch Throwable e
-        (log/error "unexpected error on parsing response body from google access token request" e)
-        nil))))
-
-(defn auth
-  [_req]
-  (let [token  (tokens/generate {:iss :google-oauth :exp (dt/in-future "15m")})
-        params {:scope scope
-                :access_type "offline"
-                :include_granted_scopes true
-                :state token
-                :response_type "code"
-                :redirect_uri (build-redirect-url)
-                :client_id (:google-client-id cfg/config)}
-        query  (uri/map->query-string params)
-        uri    (-> (uri/uri base-goauth-uri)
-                   (assoc :query query))]
-    {:status 200
-     :body {:redirect-uri (str uri)}}))
-
-
-(defn callback
-  [req]
-  (let [token (get-in req [:params :state])
-        _     (tokens/verify token {:iss :google-oauth})
-        info  (some-> (get-in req [:params :code])
-                      (get-access-token)
-                      (get-user-info))]
-
-    (when-not info
-      (ex/raise :type :authentication
-                :code :unable-to-authenticate-with-google))
-
-    (let [profile (sm/handle {::sm/type :login-or-register
-                              :email (:email info)
-                              :fullname (:fullname info)})
-          uagent  (get-in req [:headers "user-agent"])
-
-          token   (tokens/generate
-                   {:iss :auth
-                    :exp (dt/in-future "15m")
-                    :profile-id (:id profile)})
-          uri     (-> (uri/uri (:public-uri cfg/config))
-                      (assoc :path "/#/auth/verify-token")
-                      (assoc :query (uri/map->query-string {:token token})))
-          sid     (session/create (:id profile) uagent)]
-
-      {:status 302
-       :headers {"location" (str uri)}
-       :cookies (session/cookies sid)
-       :body ""})))

backend/src/app/http/auth/ldap.clj

@@ -1,80 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.auth.ldap
-  (:require
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
-   [app.http.session :as session]
-   [app.services.mutations :as sm]
-   [clj-ldap.client :as client]
-   [clojure.set :as set]
-   [clojure.string]
-   [clojure.tools.logging :as log]
-   [mount.core :refer [defstate]]))
-
-(defn replace-several [s & {:as replacements}]
-  (reduce-kv clojure.string/replace s replacements))
-
-(declare *ldap-pool)
-
-(defstate *ldap-pool
-  :start (delay
-           (try
-             (client/connect (merge {:host {:address (:ldap-auth-host cfg/config)
-                                            :port (:ldap-auth-port cfg/config)}}
-                                    (-> cfg/config
-                                        (select-keys [:ldap-auth-ssl
-                                                      :ldap-auth-starttls
-                                                      :ldap-bind-dn
-                                                      :ldap-bind-password])
-                                        (set/rename-keys {:ldap-auth-ssl :ssl?
-                                                          :ldap-auth-starttls :startTLS?
-                                                          :ldap-bind-dn  :bind-dn
-                                                          :ldap-bind-password :password}))))
-             (catch Exception e
-               (log/errorf e "Cannot connect to LDAP %s:%s"
-                             (:ldap-auth-host cfg/config) (:ldap-auth-port cfg/config)))))
-  :stop (when (realized? *ldap-pool)
-          (some-> *ldap-pool deref (.close))))
-
-(defn- auth-with-ldap [username password]
-  (when-some [conn (some-> *ldap-pool deref)]
-    (let [user-search-query (replace-several (:ldap-auth-user-query cfg/config)
-                                             "$username" username)
-          user-attributes (-> cfg/config
-                              (select-keys [:ldap-auth-username-attribute
-                                            :ldap-auth-email-attribute
-                                            :ldap-auth-fullname-attribute
-                                            :ldap-auth-avatar-attribute])
-                              vals)]
-      (when-some [user-entry (-> conn
-                                 (client/search (:ldap-auth-base-dn cfg/config)
-                                                {:filter user-search-query
-                                                 :sizelimit 1
-                                                 :attributes user-attributes})
-                                 (first))]
-        (when-not (client/bind? conn (:dn user-entry) password)
-          (ex/raise :type :authentication
-                    :code :wrong-credentials))
-        (set/rename-keys user-entry {(keyword (:ldap-auth-avatar-attribute cfg/config)) :photo
-                                     (keyword (:ldap-auth-fullname-attribute cfg/config)) :fullname
-                                     (keyword (:ldap-auth-email-attribute cfg/config)) :email})))))
-
-(defn auth [req]
-  (let [data (:body-params req)
-        uagent (get-in req [:headers "user-agent"])]
-    (when-some [info (auth-with-ldap (:email data) (:password data))]
-      (let [profile (sm/handle {::sm/type :login-or-register
-                                :email (:email info)
-                                :fullname (:fullname info)})
-            sid (session/create (:id profile) uagent)]
-        {:status 200
-         :cookies (session/cookies sid)
-         :body profile}))))

backend/src/app/http/awsns.clj

@@ -0,0 +1,196 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.awsns
+  "AWS SNS webhook handler for bounces."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.db :as db]
+   [app.db.sql :as sql]
+   [app.util.http :as http]
+   [app.util.logging :as l]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]
+   [jsonista.core :as j]))
+
+(declare parse-json)
+(declare parse-notification)
+(declare process-report)
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::handler
+  [_ cfg]
+  (fn [request]
+    (let [body  (parse-json (slurp (:body request)))
+          mtype (get body "Type")]
+      (cond
+        (= mtype "SubscriptionConfirmation")
+        (let [surl   (get body "SubscribeURL")
+              stopic (get body "TopicArn")]
+          (l/info :action "subscription received" :topic stopic :url surl)
+          (http/send! {:uri surl :method :post :timeout 10000}))
+
+        (= mtype "Notification")
+        (when-let [message (parse-json (get body "Message"))]
+          (let [notification (parse-notification cfg message)]
+            (process-report cfg notification)))
+
+        :else
+        (l/warn :hint "unexpected data received"
+                :report (pr-str body)))
+      {:status 200 :body ""})))
+
+(defn- parse-bounce
+  [data]
+  {:type        "bounce"
+   :kind        (str/lower (get data "bounceType"))
+   :category    (str/lower (get data "bounceSubType"))
+   :feedback-id (get data "feedbackId")
+   :timestamp   (get data "timestamp")
+   :recipients  (->> (get data "bouncedRecipients")
+                     (mapv (fn [item]
+                             {:email  (str/lower (get item "emailAddress"))
+                              :status (get item "status")
+                              :action (get item "action")
+                              :dcode  (get item "diagnosticCode")})))})
+
+(defn- parse-complaint
+  [data]
+  {:type          "complaint"
+   :user-agent    (get data "userAgent")
+   :kind          (get data "complaintFeedbackType")
+   :category      (get data "complaintSubType")
+   :timestamp     (get data "arrivalDate")
+   :feedback-id   (get data "feedbackId")
+   :recipients    (->> (get data "complainedRecipients")
+                       (mapv #(get % "emailAddress"))
+                       (mapv str/lower))})
+
+(defn- extract-headers
+  [mail]
+  (reduce (fn [acc item]
+            (let [key (get item "name")
+                  val (get item "value")]
+              (assoc acc (str/lower key) val)))
+          {}
+          (get mail "headers")))
+
+(defn- extract-identity
+  [{:keys [tokens] :as cfg} headers]
+  (let [tdata (get headers "x-penpot-data")]
+    (when-not (str/empty? tdata)
+      (let [result (tokens :verify {:token tdata :iss :profile-identity})]
+        (:profile-id result)))))
+
+(defn- parse-notification
+  [cfg message]
+  (let [type (get message "notificationType")
+        data (case type
+               "Bounce" (parse-bounce (get message "bounce"))
+               "Complaint" (parse-complaint (get message "complaint"))
+               {:type (keyword (str/lower type))
+                :message message})]
+    (when data
+      (let [mail (get message "mail")]
+        (when-not mail
+          (ex/raise :type :internal
+                    :code :incomplete-notification
+                    :hint "no email data received, please enable full headers report"))
+        (let [headers (extract-headers mail)
+              mail    {:destination (get mail "destination")
+                       :source      (get mail "source")
+                       :timestamp   (get mail "timestamp")
+                       :subject     (get-in mail ["commonHeaders" "subject"])
+                       :headers     headers}]
+          (assoc data
+                 :mail mail
+                 :profile-id (extract-identity cfg headers)))))))
+
+(defn- parse-json
+  [v]
+  (ex/ignoring
+   (j/read-value v)))
+
+(defn- register-bounce-for-profile
+  [{:keys [pool]} {:keys [type kind profile-id] :as report}]
+  (when (= kind "permanent")
+    (db/with-atomic [conn pool]
+      (db/insert! conn :profile-complaint-report
+                  {:profile-id profile-id
+                   :type (name type)
+                   :content (db/tjson report)})
+
+      ;; TODO: maybe also try to find profiles by mail and if exists
+      ;; register profile reports for them?
+      (doseq [recipient (:recipients report)]
+        (db/insert! conn :global-complaint-report
+                    {:email (:email recipient)
+                     :type (name type)
+                     :content (db/tjson report)}))
+
+      (let [profile (db/exec-one! conn (sql/select :profile {:id profile-id}))]
+        (when (some #(= (:email profile) (:email %)) (:recipients report))
+          ;; If the report matches the profile email, this means that
+          ;; the report is for itself, can be caused when a user
+          ;; registers with an invalid email or the user email is
+          ;; permanently rejecting receiving the email. In this case we
+          ;; have no option to mark the user as muted (and in this case
+          ;; the profile will be also inactive.
+          (db/update! conn :profile
+                      {:is-muted true}
+                      {:id profile-id}))))))
+
+(defn- register-complaint-for-profile
+  [{:keys [pool]} {:keys [type profile-id] :as report}]
+  (db/with-atomic [conn pool]
+    (db/insert! conn :profile-complaint-report
+                {:profile-id profile-id
+                 :type (name type)
+                 :content (db/tjson report)})
+
+    ;; TODO: maybe also try to find profiles by email and if exists
+    ;; register profile reports for them?
+    (doseq [email (:recipients report)]
+      (db/insert! conn :global-complaint-report
+                  {:email email
+                   :type (name type)
+                   :content (db/tjson report)}))
+
+    (let [profile (db/exec-one! conn (sql/select :profile {:id profile-id}))]
+      (when (some #(= % (:email profile)) (:recipients report))
+        ;; If the report matches the profile email, this means that
+        ;; the report is for itself, rare case but can happen; In this
+        ;; case just mark profile as muted (very rare case).
+        (db/update! conn :profile
+                    {:is-muted true}
+                    {:id profile-id})))))
+
+(defn- process-report
+  [cfg {:keys [type profile-id] :as report}]
+  (l/trace :action "procesing report" :report (pr-str report))
+  (cond
+    ;; In this case we receive a bounce/complaint notification without
+    ;; confirmed identity, we just emit a warning but do nothing about
+    ;; it because this is not a normal case. All notifications should
+    ;; come with profile identity.
+    (nil? profile-id)
+    (l/warn :msg "a notification without identity recevied from AWS"
+            :report (pr-str report))
+
+    (= "bounce" type)
+    (register-bounce-for-profile cfg report)
+
+    (= "complaint" type)
+    (register-complaint-for-profile cfg report)
+
+    :else
+    (l/warn :msg "unrecognized report received from AWS"
+            :report (pr-str report))))
+
+

backend/src/app/http/errors.clj

@@ -2,105 +2,143 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.errors
   "A errors handling for the http server."
   (:require
    [app.common.exceptions :as ex]
-   [clojure.tools.logging :as log]
+   [app.common.uuid :as uuid]
+   [app.util.logging :as l]
    [cuerdas.core :as str]
    [expound.alpha :as expound]))
 
+(defn- explain-error
+  [error]
+  (with-out-str
+    (expound/printer (:data error))))
+
+(defn get-error-context
+  [request error]
+  (let [edata (ex-data error)]
+    (merge
+     {:id      (uuid/next)
+      :path    (:uri request)
+      :method  (:request-method request)
+      :params  (:params request)
+      :data    edata}
+     (let [headers (:headers request)]
+       {:user-agent (get headers "user-agent")
+        :frontend-version (get headers "x-frontend-version" "unknown")})
+     (when (and (map? edata) (:data edata))
+       {:explain (explain-error edata)}))))
+
 (defmulti handle-exception
   (fn [err & _rest]
     (let [edata (ex-data err)]
       (or (:type edata)
           (class err)))))
 
-(defmethod handle-exception :authorization
+(defmethod handle-exception :authentication
   [err _]
-  {:status 403
-   :body (ex-data err)})
+  {:status 401 :body (ex-data err)})
+
+
+(defmethod handle-exception :restriction
+  [err _]
+  {:status 400 :body (ex-data err)})
 
 (defmethod handle-exception :validation
   [err req]
   (let [header (get-in req [:headers "accept"])
         edata  (ex-data err)]
-    (cond
-      (and (str/starts-with? header "text/html")
-           (= :spec-validation (:code edata)))
+    (if (and (= :spec-validation (:code edata))
+             (str/starts-with? header "text/html"))
       {:status 400
        :headers {"content-type" "text/html"}
        :body (str "<pre style='font-size:16px'>"
-                  (with-out-str
-                    (:data edata))
+                  (explain-error edata)
                   "</pre>\n")}
-      :else
       {:status 400
-       :body edata})))
+       :body   (cond-> edata
+                 (map? (:data edata))
+                 (-> (assoc :explain (explain-error edata))
+                     (dissoc :data)))})))
 
-(defmethod handle-exception :ratelimit
-  [_ _]
-  {:status 429
-   :headers {"retry-after" 1000}
-   :body ""})
+(defmethod handle-exception :assertion
+  [error request]
+  (let [edata (ex-data error)
+        cdata (get-error-context request error)]
+    (l/update-thread-context! cdata)
+    (l/error :hint "internal error: assertion"
+             :error-id (str (:id cdata))
+             :cause error)
+
+    {:status 500
+     :body {:type :server-error
+            :code :assertion
+            :data (-> edata
+                      (assoc :explain (explain-error edata))
+                      (dissoc :data))}}))
 
 (defmethod handle-exception :not-found
   [err _]
-  (let [response (ex-data err)]
-    {:status 404
-     :body response}))
-
-(defmethod handle-exception :service-error
-  [err req]
-  (handle-exception (.getCause ^Throwable err) req))
-
-(defmethod handle-exception :parse
-  [err _]
-  {:status 400
-   :body {:type :parse
-          :message (ex-message err)}})
-
-(defn get-context-string
-  [err request]
-  (str
-   "=| uri:          " (pr-str (:uri request)) "\n"
-   "=| method:       " (pr-str (:request-method request)) "\n"
-   "=| path-params:  " (pr-str (:path-params request)) "\n"
-   "=| query-params: " (pr-str (:query-params request)) "\n"
-
-   (when-let [bparams (:body-params request)]
-     (str "=| body-params:  " (pr-str bparams) "\n"))
-
-   (when (ex/ex-info? err)
-     (str "=| ex-data:      " (pr-str (ex-data err)) "\n"))
-
-   "\n"))
-
-
-(defmethod handle-exception :assertion
-  [err request]
-  (let [{:keys [data] :as edata} (ex-data err)]
-    (log/errorf err
-                (str "Assertion error\n"
-                     (get-context-string err request)
-                     (with-out-str (expound/printer data))))
-    {:status 500
-     :body {:type :internal-error
-            :message "Assertion error"
-            :data (ex-data err)}}))
+  {:status 404 :body (ex-data err)})
 
 (defmethod handle-exception :default
-  [err request]
-  (log/errorf err (str "Internal Error\n" (get-context-string err request)))
-  {:status 500
-   :body {:type :internal-error
-          :message (ex-message err)
-          :data (ex-data err)}})
+  [error request]
+  (let [edata (ex-data error)]
+    ;; NOTE: this is a special case for the idle-in-transaction error;
+    ;; when it happens, the connection is automatically closed and
+    ;; next-jdbc combines the two errors in a single ex-info. We only
+    ;; need the :handling error, because the :rollback error will be
+    ;; always "connection closed".
+    (if (and (ex/exception? (:rollback edata))
+             (ex/exception? (:handling edata)))
+      (handle-exception (:handling edata) request)
+      (let [cdata (get-error-context request error)]
+        (l/update-thread-context! cdata)
+        (l/error :hint "internal error"
+                 :error-message (ex-message error)
+                 :error-id (str (:id cdata))
+                 :cause error)
+        {:status 500
+         :body {:type :server-error
+                :code :unexpected
+                :hint (ex-message error)
+                :data edata}}))))
+
+(defmethod handle-exception org.postgresql.util.PSQLException
+  [error request]
+  (let [cdata (get-error-context request error)
+        state (.getSQLState ^java.sql.SQLException error)]
+
+    (l/update-thread-context! cdata)
+    (l/error :hint "psql exception"
+             :error-message (ex-message error)
+             :error-id (str (:id cdata))
+             :sql-state state
+             :cause error)
+
+    (cond
+      (= state "57014")
+      {:status 504
+       :body {:type :server-timeout
+              :code :statement-timeout
+              :hint (ex-message error)}}
+
+      (= state "25P03")
+      {:status 504
+       :body {:type :server-timeout
+              :code :idle-in-transaction-timeout
+              :hint (ex-message error)}}
+
+      :else
+      {:status 500
+       :body {:type :server-error
+              :code :psql-exception
+              :hint (ex-message error)
+              :state state}})))
 
 (defn handle
   [error req]

backend/src/app/http/feedback.clj

@@ -0,0 +1,70 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.feedback
+  "A general purpose feedback module."
+  (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.config :as cfg]
+   [app.db :as db]
+   [app.emails :as eml]
+   [app.rpc.queries.profile :as profile]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(declare send-feedback)
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::handler
+  [_ {:keys [pool] :as scfg}]
+  (let [ftoken   (cfg/get :feedback-token ::no-token)
+        enabled  (cfg/get :feedback-enabled)]
+    (fn [{:keys [profile-id] :as request}]
+      (let [token  (get-in request [:headers "x-feedback-token"])
+            params (d/merge (:params request)
+                            (:body-params request))]
+
+        (when-not enabled
+          (ex/raise :type :validation
+                    :code :feedback-disabled
+                    :hint "feedback module is disabled"))
+
+        (cond
+          (uuid? profile-id)
+          (let [profile (profile/retrieve-profile-data pool profile-id)
+                params  (assoc params :from (:email profile))]
+            (when-not (:is-muted profile)
+              (send-feedback pool profile params)))
+
+          (= token ftoken)
+          (send-feedback scfg nil params))
+
+        {:status 204 :body ""}))))
+
+(s/def ::content ::us/string)
+(s/def ::from    ::us/email)
+(s/def ::subject ::us/string)
+
+(s/def ::feedback
+  (s/keys :req-un [::from ::subject ::content]))
+
+(defn send-feedback
+  [pool profile params]
+  (let [params      (us/conform ::feedback params)
+        destination (cfg/get :feedback-destination)]
+    (eml/send! {::eml/conn pool
+                ::eml/factory eml/feedback
+                :to       destination
+                :profile  profile
+                :reply-to (:from params)
+                :email    (:from params)
+                :subject  (:subject params)
+                :content  (:content params)})
+    nil))

backend/src/app/http/handlers.clj

@@ -1,76 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.handlers
-  (:require
-   [app.common.data :as d]
-   [app.common.exceptions :as ex]
-   [app.emails :as emails]
-   [app.http.session :as session]
-   [app.services.init]
-   [app.services.mutations :as sm]
-   [app.services.queries :as sq]))
-
-(def unauthorized-services
-  #{:create-demo-profile
-    :logout
-    :profile
-    :verify-token
-    :recover-profile
-    :register-profile
-    :request-profile-recovery
-    :viewer-bundle
-    :login})
-
-(defn query-handler
-  [{:keys [profile-id] :as request}]
-  (let [type (keyword (get-in request [:path-params :type]))
-        data (assoc (:params request) ::sq/type type)
-        data (if profile-id
-               (assoc data :profile-id profile-id)
-               (dissoc data :profile-id))]
-
-    (if (or (uuid? profile-id)
-            (contains? unauthorized-services type))
-      {:status 200
-       :body (sq/handle (with-meta data {:req request}))}
-      {:status 403
-       :body {:type :authentication
-              :code :unauthorized}})))
-
-(defn mutation-handler
-  [{:keys [profile-id] :as request}]
-  (let [type (keyword (get-in request [:path-params :type]))
-        data (d/merge (:params request)
-                      (:body-params request)
-                      (:uploads request)
-                      {::sm/type type})
-        data (if profile-id
-               (assoc data :profile-id profile-id)
-               (dissoc data :profile-id))]
-
-    (if (or (uuid? profile-id)
-            (contains? unauthorized-services type))
-      (let [result (sm/handle (with-meta data {:req request}))
-            mdata  (meta result)
-            resp   {:status (if (nil? (seq result)) 204 200)
-                    :body result}]
-        (cond->> resp
-          (:transform-response mdata) ((:transform-response mdata) request)))
-      {:status 403
-       :body {:type :authentication
-              :code :unauthorized}})))
-
-(defn echo-handler
-  [req]
-  {:status 200
-   :body {:params (:params req)
-          :cookies (:cookies req)
-          :headers (:headers req)}})
-

backend/src/app/http/middleware.clj

@@ -2,39 +2,72 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.middleware
   (:require
-   [app.common.exceptions :as ex]
-   [app.config :as cfg]
    [app.metrics :as mtx]
+   [app.util.json :as json]
+   [app.util.logging :as l]
    [app.util.transit :as t]
+   [buddy.core.codecs :as bc]
+   [buddy.core.hash :as bh]
+   [clojure.java.io :as io]
    [ring.middleware.cookies :refer [wrap-cookies]]
    [ring.middleware.keyword-params :refer [wrap-keyword-params]]
    [ring.middleware.multipart-params :refer [wrap-multipart-params]]
-   [ring.middleware.params :refer [wrap-params]]
-   [ring.middleware.resource :refer [wrap-resource]]))
+   [ring.middleware.params :refer [wrap-params]]))
 
-(defn- wrap-parse-request-body
+(defn wrap-server-timing
   [handler]
-  (letfn [(parse-body [body]
+  (let [seconds-from #(float (/ (- (System/nanoTime) %) 1000000000))]
+    (fn [request]
+      (let [start    (System/nanoTime)
+            response (handler request)]
+        (update response :headers
+                (fn [headers]
+                  (assoc headers "Server-Timing" (str "total;dur=" (seconds-from start)))))))))
+
+(defn wrap-parse-request-body
+  [handler]
+  (letfn [(parse-transit [body]
+            (let [reader (t/reader body)]
+              (t/read! reader)))
+
+          (parse-json [body]
+            (let [reader (io/reader body)]
+              (json/read reader)))
+
+          (parse [type body]
             (try
-              (let [reader (t/reader body)]
-                (t/read! reader))
+              (case type
+                :json (parse-json body)
+                :transit (parse-transit body))
               (catch Exception e
-                (ex/raise :type :parse
-                          :message "Unable to parse transit from request body."
-                          :cause e))))]
-    (fn [{:keys [headers body request-method] :as request}]
-      (handler
-       (cond-> request
-         (and (= "application/transit+json" (get headers "content-type"))
-              (not= request-method :get))
-         (assoc :body-params (parse-body body)))))))
+                (let [data {:type :parse
+                            :hint "unable to parse request body"
+                            :message (ex-message e)}]
+                  {:status 400
+                   :headers {"content-type" "application/transit+json"}
+                   :body (t/encode-str data {:type :json-verbose})}))))]
+
+    (fn [{:keys [headers body] :as request}]
+      (let [ctype (get headers "content-type")]
+        (handler
+         (case ctype
+           "application/transit+json"
+           (let [params (parse :transit body)]
+             (-> request
+                 (assoc :body-params params)
+                 (update :params merge params)))
+
+           "application/json"
+           (let [params (parse :json body)]
+             (-> request
+                 (assoc :body-params params)
+                 (update :params merge params)))
+
+           request))))))
 
 (def parse-request-body
   {:name ::parse-request-body
@@ -43,9 +76,7 @@
 (defn- impl-format-response-body
   [response]
   (let [body (:body response)
-        type (if (:debug-humanize-transit cfg/config)
-               :json-verbose
-               :json)]
+        type :json-verbose]
     (cond
       (coll? body)
       (-> response
@@ -71,7 +102,7 @@
   {:name ::format-response-body
    :compile (constantly wrap-format-response-body)})
 
-(defn- wrap-errors
+(defn wrap-errors
   [handler on-error]
   (fn [request]
     (try
@@ -88,7 +119,6 @@
    :wrap (fn [handler]
            (mtx/wrap-counter handler {:id "http__requests_counter"
                                       :help "Absolute http requests counter."}))})
-
 (def cookies
   {:name ::cookies
    :compile (constantly wrap-cookies)})
@@ -105,33 +135,46 @@
   {:name ::keyword-params
    :compile (constantly wrap-keyword-params)})
 
-(defn- wrap-development-cors
+(def server-timing
+  {:name ::server-timing
+   :compile (constantly wrap-server-timing)})
+
+(defn wrap-etag
   [handler]
-  (letfn [(add-cors-headers [response]
-            (update response :headers
-                    (fn [headers]
-                      (-> headers
-                          (assoc "access-control-allow-origin" "http://localhost:3449")
-                          (assoc "access-control-allow-methods" "GET,POST,DELETE,OPTIONS,PUT,HEAD,PATCH")
-                          (assoc "access-control-allow-credentials" "true")
-                          (assoc "access-control-expose-headers" "x-requested-with, content-type, cookie")
-                          (assoc "access-control-allow-headers" "content-type")))))]
+  (letfn [(generate-etag [{:keys [body] :as response}]
+            (str "W/\"" (-> body bh/blake2b-128 bc/bytes->hex) "\""))
+          (get-match [{:keys [headers] :as request}]
+            (get headers "if-none-match"))]
     (fn [request]
-      (if (= (:request-method request) :options)
-        (-> {:status 200 :body ""}
-            (add-cors-headers))
-        (let [response (handler request)]
-          (add-cors-headers response))))))
+      (let [response (handler request)]
+        (if (= :get (:request-method request))
+          (let [etag     (generate-etag response)
+                match    (get-match request)
+                response (update response :headers #(assoc % "ETag" etag))]
+            (cond-> response
+              (and (string? match)
+                   (= :get (:request-method request))
+                   (= etag match))
+              (-> response
+                  (assoc :body "")
+                  (assoc :status 304))))
+          response)))))
 
-(def development-cors
-  {:name ::development-cors
-   :compile (fn [& _args]
-              (when *assert*
-                wrap-development-cors))})
-
-(def development-resources
-  {:name ::development-resources
-   :compile (fn [& _args]
-              (when *assert*
-                #(wrap-resource % "public")))})
+(def etag
+  {:name ::etag
+   :compile (constantly wrap-etag)})
 
+(defn activity-logger
+  [handler]
+  (let [logger "penpot.profile-activity"]
+    (fn [{:keys [headers] :as request}]
+      (let [ip-addr    (get headers "x-forwarded-for")
+            profile-id (:profile-id request)
+            qstring    (:query-string request)]
+        (l/info ::l/async true
+                ::l/logger logger
+                :ip-addr ip-addr
+                :profile-id profile-id
+                :uri (str (:uri request) (when qstring (str "?" qstring)))
+                :method (name (:request-method request)))
+        (handler request)))))

backend/src/app/http/oauth.clj

@@ -0,0 +1,341 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.http.oauth
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uri :as u]
+   [app.config :as cf]
+   [app.util.http :as http]
+   [app.util.logging :as l]
+   [app.util.time :as dt]
+   [clojure.data.json :as json]
+   [clojure.set :as set]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+(defn redirect-response
+  [uri]
+  {:status 302
+   :headers {"location" (str uri)}
+   :body ""})
+
+(defn generate-error-redirect-uri
+  [cfg]
+  (-> (u/uri (:public-uri cfg))
+      (assoc :path "/#/auth/login")
+      (assoc :query (u/map->query-string {:error "unable-to-auth"}))))
+
+(defn register-profile
+  [{:keys [rpc] :as cfg} info]
+  (let [method-fn (get-in rpc [:methods :mutation :login-or-register])
+        profile   (method-fn info)]
+    (cond-> profile
+      (some? (:invitation-token info))
+      (assoc :invitation-token (:invitation-token info)))))
+
+(defn generate-redirect-uri
+  [{:keys [tokens] :as cfg} profile]
+  (let [token (or (:invitation-token profile)
+                  (tokens :generate {:iss :auth
+                                     :exp (dt/in-future "15m")
+                                     :profile-id (:id profile)}))]
+    (-> (u/uri (:public-uri cfg))
+        (assoc :path "/#/auth/verify-token")
+        (assoc :query (u/map->query-string {:token token})))))
+
+(defn- build-redirect-uri
+  [{:keys [provider] :as cfg}]
+  (let [public (u/uri (:public-uri cfg))]
+    (str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
+
+(defn- build-auth-uri
+  [{:keys [provider] :as cfg} state]
+  (let [params {:client_id (:client-id provider)
+                :redirect_uri (build-redirect-uri cfg)
+                :response_type "code"
+                :state state
+                :scope (str/join " " (:scopes provider []))}
+        query  (u/map->query-string params)]
+    (-> (u/uri (:auth-uri provider))
+        (assoc :query query)
+        (str))))
+
+(defn retrieve-access-token
+  [{:keys [provider] :as cfg} code]
+  (try
+    (let [params {:client_id (:client-id provider)
+                  :client_secret (:client-secret provider)
+                  :code code
+                  :grant_type "authorization_code"
+                  :redirect_uri (build-redirect-uri cfg)}
+          req    {:method :post
+                  :headers {"content-type" "application/x-www-form-urlencoded"}
+                  :uri (:token-uri provider)
+                  :body (u/map->query-string params)}
+          res    (http/send! req)]
+      (when (= 200 (:status res))
+        (let [data (json/read-str (:body res))]
+          {:token (get data "access_token")
+           :type  (get data "token_type")})))
+    (catch Exception e
+      (l/error :hint "unexpected error on retrieve-access-token"
+               :cause e)
+      nil)))
+
+(defn- retrieve-user-info
+  [{:keys [provider] :as cfg} tdata]
+  (try
+    (let [req {:uri (:user-uri provider)
+               :headers {"Authorization" (str (:type tdata) " " (:token tdata))}
+               :timeout 6000
+               :method :get}
+          res (http/send! req)]
+
+      (when (= 200 (:status res))
+        (let [info (json/read-str (:body res) :key-fn keyword)]
+          {:backend (:name provider)
+           :email (:email info)
+           :fullname (:name info)
+           :props (dissoc info :name :email)})))
+
+    (catch Exception e
+      (l/error :hint "unexpected exception on retrieve-user-info"
+               :cause e)
+      nil)))
+
+(s/def ::backend ::us/not-empty-string)
+(s/def ::email ::us/not-empty-string)
+(s/def ::fullname ::us/not-empty-string)
+(s/def ::props (s/map-of ::us/keyword any?))
+
+(s/def ::info
+  (s/keys :req-un [::backend
+                   ::email
+                   ::fullname
+                   ::props]))
+
+(defn retrieve-info
+  [{:keys [tokens provider] :as cfg} request]
+  (let [state  (get-in request [:params :state])
+        state  (tokens :verify {:token state :iss :oauth})
+        info   (some->> (get-in request [:params :code])
+                        (retrieve-access-token cfg)
+                        (retrieve-user-info cfg))]
+
+    (when-not (s/valid? ::info info)
+      (l/warn :hint "received incomplete profile info object (please set correct scopes)"
+              :info (pr-str info))
+      (ex/raise :type :internal
+                :code :unable-to-auth
+                :hint "no user info"))
+
+    ;; If the provider is OIDC, we can proceed to check
+    ;; roles if they are defined.
+    (when (and (= "oidc" (:name provider))
+               (seq (:roles provider)))
+      (let [provider-roles (into #{} (:roles provider))
+            profile-roles  (let [attr  (cf/get :oidc-roles-attr :roles)
+                                 roles (get info attr)]
+                             (cond
+                               (string? roles) (into #{} (str/words roles))
+                               (vector? roles) (into #{} roles)
+                               :else #{}))]
+        ;; check if profile has a configured set of roles
+        (when-not (set/subset? provider-roles profile-roles)
+          (ex/raise :type :internal
+                    :code :unable-to-auth
+                    :hint "not enought permissions"))))
+
+    (cond-> info
+      (some? (:invitation-token state))
+      (assoc :invitation-token (:invitation-token state))
+
+      ;; If state token comes with props, merge them. The state token
+      ;; props can contain pm_ and utm_ prefixed query params.
+      (map? (:props state))
+      (update :props merge (:props state)))))
+
+;; --- HTTP HANDLERS
+
+(defn extract-props
+  [params]
+  (reduce-kv (fn [params k v]
+               (let [sk (name k)]
+                 (cond-> params
+                   (or (str/starts-with? sk "pm_")
+                       (str/starts-with? sk "pm-")
+                       (str/starts-with? sk "utm_"))
+                   (assoc (-> sk str/kebab keyword) v))))
+             {}
+             params))
+
+(defn- auth-handler
+  [{:keys [tokens] :as cfg} {:keys [params] :as request}]
+  (let [invitation (:invitation-token params)
+        props      (extract-props params)
+        state      (tokens :generate
+                           {:iss :oauth
+                            :invitation-token invitation
+                            :props props
+                            :exp (dt/in-future "15m")})
+        uri        (build-auth-uri cfg state)]
+    {:status 200
+     :body {:redirect-uri uri}}))
+
+(defn- callback-handler
+  [{:keys [session] :as cfg} request]
+  (try
+    (let [info    (retrieve-info cfg request)
+          profile (register-profile cfg info)
+          uri     (generate-redirect-uri cfg profile)
+          sxf     ((:create session) (:id profile))]
+      (->> (redirect-response uri)
+           (sxf request)))
+    (catch Exception _e
+      (-> (generate-error-redirect-uri cfg)
+          (redirect-response)))))
+
+;; --- INIT
+
+(declare initialize)
+
+(s/def ::public-uri ::us/not-empty-string)
+(s/def ::session map?)
+(s/def ::tokens fn?)
+(s/def ::rpc map?)
+
+(defmethod ig/pre-init-spec :app.http.oauth/handlers [_]
+  (s/keys :req-un [::public-uri ::session ::tokens ::rpc]))
+
+(defn wrap-handler
+  [cfg handler]
+  (fn [request]
+    (let [provider (get-in request [:path-params :provider])
+          provider (get-in @cfg [:providers provider])]
+      (when-not provider
+        (ex/raise :type :not-found
+                  :context {:provider provider}
+                  :hint "provider not configured"))
+      (-> (assoc @cfg :provider provider)
+          (handler request)))))
+
+(defmethod ig/init-key :app.http.oauth/handlers
+  [_ cfg]
+  (let [cfg (initialize cfg)]
+    {:handler (wrap-handler cfg auth-handler)
+     :callback-handler (wrap-handler cfg callback-handler)}))
+
+(defn- discover-oidc-config
+  [{:keys [base-uri] :as opts}]
+  (let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
+        response      (http/send! {:method :get :uri (str discovery-uri)})]
+    (when (= 200 (:status response))
+      (let [data (json/read-str (:body response))]
+        (assoc opts
+               :token-uri (get data "token_endpoint")
+               :auth-uri (get data "authorization_endpoint")
+               :user-uri (get data "userinfo_endpoint"))))))
+
+(defn- obfuscate-string
+  [s]
+  (if (< (count s) 10)
+    (apply str (take (count s) (repeat "*")))
+    (str (subs s 0 5)
+         (apply str (take (- (count s) 5) (repeat "*"))))))
+
+(defn- initialize-oidc-provider
+  [cfg]
+  (let [opts {:base-uri      (cf/get :oidc-base-uri)
+              :client-id     (cf/get :oidc-client-id)
+              :client-secret (cf/get :oidc-client-secret)
+              :token-uri     (cf/get :oidc-token-uri)
+              :auth-uri      (cf/get :oidc-auth-uri)
+              :user-uri      (cf/get :oidc-user-uri)
+              :scopes        (cf/get :oidc-scopes #{"openid" "profile" "email"})
+              :roles-attr    (cf/get :oidc-roles-attr)
+              :roles         (cf/get :oidc-roles)
+              :name          "oidc"}]
+    (if (and (string? (:base-uri opts))
+             (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (if (and (string? (:token-uri opts))
+               (string? (:user-uri opts))
+               (string? (:auth-uri opts)))
+        (do
+          (l/info :action "initialize" :provider "oidc" :method "static"
+                  :opts (pr-str (update opts :client-secret obfuscate-string)))
+          (assoc-in cfg [:providers "oidc"] opts))
+        (let [opts (discover-oidc-config opts)]
+          (l/info :action "initialize" :provider "oidc" :method "discover"
+                  :opts (pr-str (update opts :client-secret obfuscate-string)))
+          (assoc-in cfg [:providers "oidc"] opts)))
+      cfg)))
+
+(defn- initialize-google-provider
+  [cfg]
+  (let [opts {:client-id     (cf/get :google-client-id)
+              :client-secret (cf/get :google-client-secret)
+              :scopes        #{"openid" "email" "profile"}
+              :auth-uri      "https://accounts.google.com/o/oauth2/v2/auth"
+              :token-uri     "https://oauth2.googleapis.com/token"
+              :user-uri      "https://openidconnect.googleapis.com/v1/userinfo"
+              :name          "google"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "google"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "google"] opts))
+      cfg)))
+
+(defn- initialize-github-provider
+  [cfg]
+  (let [opts {:client-id     (cf/get :github-client-id)
+              :client-secret (cf/get :github-client-secret)
+              :scopes        #{"read:user" "user:email"}
+              :auth-uri      "https://github.com/login/oauth/authorize"
+              :token-uri     "https://github.com/login/oauth/access_token"
+              :user-uri      "https://api.github.com/user"
+              :name          "github"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "github"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "github"] opts))
+      cfg)))
+
+
+(defn- initialize-gitlab-provider
+  [cfg]
+  (let [base (cf/get :gitlab-base-uri "https://gitlab.com")
+        opts {:base-uri      base
+              :client-id     (cf/get :gitlab-client-id)
+              :client-secret (cf/get :gitlab-client-secret)
+              :scopes        #{"read_user"}
+              :auth-uri      (str base "/oauth/authorize")
+              :token-uri     (str base "/oauth/token")
+              :user-uri      (str base "/api/v4/user")
+              :name          "gitlab"}]
+    (if (and (string? (:client-id opts))
+             (string? (:client-secret opts)))
+      (do
+        (l/info :action "initialize" :provider "gitlab"
+                :opts (pr-str (update opts :client-secret obfuscate-string)))
+        (assoc-in cfg [:providers "gitlab"] opts))
+      cfg)))
+
+(defn- initialize
+  [cfg]
+  (let [cfg (agent cfg :error-mode :continue)]
+    (send-off cfg initialize-google-provider)
+    (send-off cfg initialize-gitlab-provider)
+    (send-off cfg initialize-github-provider)
+    (send-off cfg initialize-oidc-provider)
+    cfg))

backend/src/app/http/session.clj

@@ -2,65 +2,192 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-;; TODO: move to services.
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.http.session
   (:require
+   [app.common.data :as d]
+   [app.common.exceptions :as ex]
+   [app.config :as cfg]
    [app.db :as db]
-   [buddy.core.codecs :as bc]
-   [buddy.core.nonce :as bn]))
+   [app.metrics :as mtx]
+   [app.util.async :as aa]
+   [app.util.logging :as l]
+   [app.util.time :as dt]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
 
-(defn next-token
-  [n]
-  (-> (bn/random-nonce n)
-      (bc/bytes->b64u)
-      (bc/bytes->str)))
+;; A default cookie name for storing the session. We don't allow
+;; configure it.
+(def cookie-name "auth-token")
 
-(defn extract-auth-token
-  [request]
-  (get-in request [:cookies "auth-token" :value]))
+;; --- IMPL
 
-(defn retrieve
-  [conn token]
-  (when token
-    (-> (db/exec-one! conn ["select profile_id from http_session where id = ?" token])
-        (:profile-id))))
+(defn- create-session
+  [{:keys [conn tokens] :as cfg} {:keys [profile-id headers] :as request}]
+  (let [token  (tokens :generate {:iss "authentication"
+                                  :iat (dt/now)
+                                  :uid profile-id})
+        params {:user-agent (get headers "user-agent")
+                :profile-id profile-id
+                :id token}]
+    (db/insert! conn :http-session params)))
 
-(defn retrieve-from-request
-  [conn request]
-  (->> (extract-auth-token request)
-       (retrieve conn)))
-
-(defn create
-  [profile-id user-agent]
-  (let [id (next-token 64)]
-    (db/insert! db/pool :http-session {:id id
-                                       :profile-id profile-id
-                                       :user-agent user-agent})
-    id))
-
-(defn delete
-  [token]
-  (db/delete! db/pool :http-session {:id token})
+(defn- delete-session
+  [{:keys [conn] :as cfg} {:keys [cookies] :as request}]
+  (when-let [token (get-in cookies [cookie-name :value])]
+    (db/delete! conn :http-session {:id token}))
   nil)
 
-(defn cookies
-  ([id] (cookies id {}))
-  ([id opts]
-   {"auth-token" (merge opts {:value id :path "/" :http-only true})}))
+(defn- retrieve-session
+  [{:keys [conn] :as cfg} id]
+  (when id
+    (db/exec-one! conn ["select id, profile_id from http_session where id = ?" id])))
 
-(defn wrap-session
-  [handler]
+(defn- retrieve-from-request
+  [cfg {:keys [cookies] :as request}]
+  (->> (get-in cookies [cookie-name :value])
+       (retrieve-session cfg)))
+
+(defn- add-cookies
+  [response {:keys [id] :as session}]
+  (assoc response :cookies {cookie-name {:path "/" :http-only true :value id}}))
+
+(defn- clear-cookies
+  [response]
+  (assoc response :cookies {cookie-name {:value "" :max-age -1}}))
+
+(defn- middleware
+  [cfg handler]
   (fn [request]
-    (if-let [profile-id (retrieve-from-request db/pool request)]
-      (handler (assoc request :profile-id profile-id))
+    (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
+      (do
+        (a/>!! (::events-ch cfg) id)
+        (l/update-thread-context! {:profile-id profile-id})
+        (handler (assoc request :profile-id profile-id)))
       (handler request))))
 
-(def middleware
-  {:nane ::middleware
-   :compile (constantly wrap-session)})
+;; --- STATE INIT: SESSION
+
+(defmethod ig/pre-init-spec ::session [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/prep-key ::session
+  [_ cfg]
+  (d/merge {:buffer-size 64}
+           (d/without-nils cfg)))
+
+(defmethod ig/init-key ::session
+  [_ {:keys [pool] :as cfg}]
+  (let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
+        cfg    (-> cfg
+                   (assoc :conn pool)
+                   (assoc ::events-ch events))]
+    (-> cfg
+        (assoc :middleware #(middleware cfg %))
+        (assoc :create (fn [profile-id]
+                         (fn [request response]
+                           (let [request (assoc request :profile-id profile-id)
+                                 session (create-session cfg request)]
+                             (add-cookies response session)))))
+        (assoc :delete (fn [request response]
+                         (delete-session cfg request)
+                         (-> response
+                             (assoc :status 204)
+                             (assoc :body "")
+                             (clear-cookies)))))))
+
+(defmethod ig/halt-key! ::session
+  [_ data]
+  (a/close! (::events-ch data)))
+
+
+;; --- STATE INIT: SESSION UPDATER
+
+(declare update-sessions)
+
+(s/def ::session map?)
+(s/def ::max-batch-age ::cfg/http-session-updater-batch-max-age)
+(s/def ::max-batch-size ::cfg/http-session-updater-batch-max-size)
+
+(defmethod ig/pre-init-spec ::updater [_]
+  (s/keys :req-un [::db/pool ::wrk/executor ::mtx/metrics ::session]
+          :opt-un [::max-batch-age
+                   ::max-batch-size]))
+
+(defmethod ig/prep-key ::updater
+  [_ cfg]
+  (merge {:max-batch-age (dt/duration {:minutes 5})
+          :max-batch-size 200}
+         (d/without-nils cfg)))
+
+(defmethod ig/init-key ::updater
+  [_ {:keys [session metrics] :as cfg}]
+  (l/info :action "initialize session updater"
+          :max-batch-age (str (:max-batch-age cfg))
+          :max-batch-size (str (:max-batch-size cfg)))
+  (let [input (aa/batch (::events-ch session)
+                        {:max-batch-size (:max-batch-size cfg)
+                         :max-batch-age (inst-ms (:max-batch-age cfg))})
+        mcnt  (mtx/create
+               {:name "http_session_update_total"
+                :help "A counter of session update batch events."
+                :registry (:registry metrics)
+                :type :counter})]
+    (a/go-loop []
+      (when-let [[reason batch] (a/<! input)]
+        (let [result (a/<! (update-sessions cfg batch))]
+          (mcnt :inc)
+          (cond
+            (ex/exception? result)
+            (l/error :task "updater"
+                     :hint "unexpected error on update sessions"
+                     :cause result)
+
+            (= :size reason)
+            (l/debug :task "updater"
+                     :action "update sessions"
+                     :reason (name reason)
+                     :count result))
+          (recur))))))
+
+(defn- update-sessions
+  [{:keys [pool executor]} ids]
+  (aa/with-thread executor
+    (db/exec-one! pool ["update http_session set updated_at=now() where id = ANY(?)"
+                        (into-array String ids)])
+    (count ids)))
+
+;; --- STATE INIT: SESSION GC
+
+(declare sql:delete-expired)
+
+(s/def ::max-age ::dt/duration)
+
+(defmethod ig/pre-init-spec ::gc-task [_]
+  (s/keys :req-un [::db/pool]
+          :opt-un [::max-age]))
+
+(defmethod ig/prep-key ::gc-task
+  [_ cfg]
+  (merge {:max-age (dt/duration {:days 2})}
+         (d/without-nils cfg)))
+
+(defmethod ig/init-key ::gc-task
+  [_ {:keys [pool max-age] :as cfg}]
+  (fn [_]
+    (db/with-atomic [conn pool]
+      (let [interval (db/interval max-age)
+            result   (db/exec-one! conn [sql:delete-expired interval])
+            result   (:next.jdbc/update-count result)]
+        (l/debug :task "gc"
+                 :action "clean http sessions"
+                 :count result)
+        result))))
+
+(def ^:private
+  sql:delete-expired
+  "delete from http_session
+    where updated_at < now() - ?::interval")

backend/src/app/http/ws.clj

@@ -1,61 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.http.ws
-  "Web Socket handlers"
-  (:require
-   [app.common.spec :as us]
-   [app.db :as db]
-   [app.http.session :refer [wrap-session]]
-   [app.services.notifications :as nf]
-   [clojure.spec.alpha :as s]
-   [ring.middleware.cookies :refer [wrap-cookies]]
-   [ring.middleware.keyword-params :refer [wrap-keyword-params]]
-   [ring.middleware.params :refer [wrap-params]]))
-
-(s/def ::file-id ::us/uuid)
-(s/def ::session-id ::us/uuid)
-
-(s/def ::websocket-params
-  (s/keys :req-un [::file-id ::session-id]))
-
-(def sql:retrieve-file
-  "select f.id as id,
-          p.team_id as team_id
-     from file as f
-     join project as p on (p.id = f.project_id)
-    where f.id = ?")
-
-(defn retrieve-file
-  [conn id]
-  (db/exec-one! conn [sql:retrieve-file id]))
-
-(defn websocket
-  [{:keys [profile-id] :as req}]
-  (let [params (us/conform ::websocket-params (:params req))
-        file   (retrieve-file db/pool (:file-id params))
-        params (assoc params
-                      :profile-id profile-id
-                      :team-id (:team-id file))]
-    (cond
-      (not profile-id)
-      {:error {:code 403 :message "Authentication required"}}
-
-      (not file)
-      {:error {:code 404 :message "File does not exists"}}
-
-      :else
-      (nf/websocket params))))
-
-(def handler
-  (-> websocket
-      (wrap-session)
-      (wrap-keyword-params)
-      (wrap-cookies)
-      (wrap-params)))

backend/src/app/loggers/audit.clj

@@ -0,0 +1,229 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.audit
+  "Services related to the user activity (audit log)."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.logging :as l]
+   [app.util.time :as dt]
+   [app.util.transit :as t]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]
+   [lambdaisland.uri :as u]))
+
+(defn clean-props
+  [{:keys [profile-id] :as event}]
+  (letfn [(clean-common [props]
+            (-> props
+                (dissoc :session-id)
+                (dissoc :password)
+                (dissoc :old-password)
+                (dissoc :token)))
+
+          (clean-profile-id [props]
+            (cond-> props
+              (= profile-id (:profile-id props))
+              (dissoc :profile-id)))
+
+          (clean-complex-data [props]
+            (reduce-kv (fn [props k v]
+                         (cond-> props
+                           (or (string? v)
+                               (uuid? v)
+                               (boolean? v)
+                               (number? v))
+                           (assoc k v)
+
+                           (keyword? v)
+                           (assoc k (name v))))
+                       {}
+                       props))]
+    (update event :props #(-> % clean-common clean-profile-id clean-complex-data))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Collector
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Defines a service that collects the audit/activity log using
+;; internal database. Later this audit log can be transferred to
+;; an external storage and data cleared.
+
+(declare persist-events)
+(s/def ::enabled ::us/boolean)
+
+(defmethod ig/pre-init-spec ::collector [_]
+  (s/keys :req-un [::db/pool ::wrk/executor ::enabled]))
+
+(def event-xform
+  (comp
+   (filter :profile-id)
+   (map clean-props)))
+
+(defmethod ig/init-key ::collector
+  [_ {:keys [enabled] :as cfg}]
+  (when enabled
+    (l/info :msg "intializing audit collector")
+    (let [input  (a/chan 512 event-xform)
+          buffer (aa/batch input {:max-batch-size 100
+                                  :max-batch-age (* 10 1000) ; 10s
+                                  :init []})]
+      (a/go-loop []
+        (when-let [[_type events] (a/<! buffer)]
+          (let [res (a/<! (persist-events cfg events))]
+            (when (ex/exception? res)
+              (l/error :hint "error on persiting events"
+                       :cause res)))
+          (recur)))
+
+      (fn [& [cmd & params]]
+        (case cmd
+          :stop (a/close! input)
+          :submit (when-not (a/offer! input (first params))
+                    (l/warn :msg "activity channel is full")))))))
+
+
+(defn- persist-events
+  [{:keys [pool executor] :as cfg} events]
+  (letfn [(event->row [event]
+            [(uuid/next)
+             (:name event)
+             (:type event)
+             (:profile-id event)
+             (db/tjson (:props event))])]
+
+    (aa/with-thread executor
+      (db/with-atomic [conn pool]
+        (db/insert-multi! conn :audit-log
+                          [:id :name :type :profile-id :props]
+                          (sequence (map event->row) events))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Archive Task
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; This is a task responsible to send the accomulated events to an
+;; external service for archival.
+
+(declare archive-events)
+
+(s/def ::uri ::us/string)
+(s/def ::tokens fn?)
+
+(defmethod ig/pre-init-spec ::archive-task [_]
+  (s/keys :req-un [::db/pool ::tokens ::enabled]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::archive-task
+  [_ {:keys [uri enabled] :as cfg}]
+  (fn [_]
+    (when (and enabled (not uri))
+      (ex/raise :type :internal
+                :code :task-not-configured
+                :hint "archive task not configured, missing uri"))
+    (loop []
+      (let [res (archive-events cfg)]
+        (when (= res :continue)
+          (aa/thread-sleep 200)
+          (recur))))))
+
+(def sql:retrieve-batch-of-audit-log
+  "select * from audit_log
+    where archived_at is null
+    order by created_at asc
+    limit 100
+      for update skip locked;")
+
+(defn archive-events
+  [{:keys [pool uri tokens] :as cfg}]
+  (letfn [(decode-row [{:keys [props] :as row}]
+            (cond-> row
+              (db/pgobject? props)
+              (assoc :props (db/decode-transit-pgobject props))))
+
+          (row->event [{:keys [name type created-at profile-id props]}]
+            {:type type
+             :name name
+             :timestamp created-at
+             :profile-id profile-id
+             :props props})
+
+          (send [events]
+            (let [token   (tokens :generate {:iss "authentication"
+                                             :iat (dt/now)
+                                             :uid uuid/zero})
+                  body    (t/encode {:events events})
+                  headers {"content-type" "application/transit+json"
+                           "origin" (cf/get :public-uri)
+                           "cookie" (u/map->query-string {:auth-token token})}
+                  params  {:uri uri
+                           :timeout 5000
+                           :method :post
+                           :headers headers
+                           :body body}
+                  resp    (http/send! params)]
+              (when (not= (:status resp) 204)
+                (ex/raise :type :internal
+                          :code :unable-to-send-events
+                          :hint "unable to send events"
+                          :context resp))))
+
+          (mark-as-archived [conn rows]
+            (db/exec-one! conn ["update audit_log set archived_at=now() where id = ANY(?)"
+                                (->> (map :id rows)
+                                     (into-array java.util.UUID)
+                                     (db/create-array conn "uuid"))]))]
+
+    (db/with-atomic [conn pool]
+      (let [rows   (db/exec! conn [sql:retrieve-batch-of-audit-log])
+
+            xform  (comp (map decode-row)
+                         (map row->event))
+            events (into [] xform rows)]
+        (l/debug :action "archive-events" :uri uri :events (count events))
+        (if (empty? events)
+          :empty
+          (do
+            (send events)
+            (mark-as-archived conn rows)
+            :continue))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GC Task
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare clean-archived)
+
+(s/def ::max-age ::cf/audit-archive-gc-max-age)
+
+(defmethod ig/pre-init-spec ::archive-gc-task [_]
+  (s/keys :req-un [::db/pool ::enabled ::max-age]))
+
+(defmethod ig/init-key ::archive-gc-task
+  [_ cfg]
+  (fn [_]
+    (clean-archived cfg)))
+
+(def sql:clean-archived
+  "delete from audit_log
+    where archived_at is not null
+      and archived_at < now() - ?::interval")
+
+(defn- clean-archived
+  [{:keys [pool max-age]}]
+  (let [interval (db/interval max-age)
+        result   (db/exec-one! pool [sql:clean-archived interval])
+        result   (:next.jdbc/update-count result)]
+    (l/debug :action "clean archived audit log" :removed result)
+    result))

backend/src/app/loggers/loki.clj

@@ -0,0 +1,99 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.loki
+  "A Loki integration."
+  (:require
+   [app.common.spec :as us]
+   [app.config :as cfg]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.json :as json]
+   [app.util.logging :as l]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig]))
+
+(declare handle-event)
+
+(s/def ::uri ::us/string)
+(s/def ::receiver fn?)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::receiver]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver uri] :as cfg}]
+  (when uri
+    (l/info :msg "intializing loki reporter" :uri uri)
+    (let [input (a/chan (a/dropping-buffer 512))]
+      (receiver :sub input)
+      (a/go-loop []
+        (let [msg (a/<! input)]
+          (if (nil? msg)
+            (l/info :msg "stoping error reporting loop")
+            (do
+              (a/<! (handle-event cfg msg))
+              (recur)))))
+      input)))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (when output
+    (a/close! output)))
+
+(defn- prepare-payload
+  [event]
+  (let [labels {:host    (cfg/get :host)
+                :tenant  (cfg/get :tenant)
+                :version (:full cfg/version)
+                :logger  (:logger event)
+                :level   (:level event)}]
+    {:streams
+     [{:stream labels
+       :values [[(str (* (inst-ms (:created-at event)) 1000000))
+                 (str (:message event)
+                      (when-let [error (:error event)]
+                        (str "\n" (:trace error))))]]}]}))
+
+(defn- send-log
+  [uri payload i]
+  (try
+    (let [response (http/send! {:uri uri
+                                :timeout 6000
+                                :method :post
+                                :headers {"content-type" "application/json"}
+                                :body (json/encode payload)})]
+      (cond
+        (= (:status response) 204)
+        true
+
+        (= (:status response) 400)
+        (do
+          (l/error :hint "error on sending log to loki (no retry)"
+                   :rsp (pr-str response))
+          true)
+
+        :else
+        (do
+          (l/error :hint "error on sending log to loki" :try i
+                   :rsp (pr-str response))
+          false)))
+    (catch Exception e
+      (l/error :hint "error on sending message to loki" :cause e :try i)
+      false)))
+
+(defn- handle-event
+  [{:keys [executor uri]} event]
+  (aa/with-thread executor
+    (let [payload (prepare-payload event)]
+      (loop [i 1]
+        (when (and (not (send-log uri payload i)) (< i 20))
+          (Thread/sleep (* i 2000))
+          (recur (inc i)))))))
+

backend/src/app/loggers/mattermost.clj

@@ -0,0 +1,153 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.mattermost
+  "A mattermost integration for error reporting."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.config :as cfg]
+   [app.db :as db]
+   [app.util.async :as aa]
+   [app.util.http :as http]
+   [app.util.json :as json]
+   [app.util.logging :as l]
+   [app.util.template :as tmpl]
+   [app.worker :as wrk]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Error Listener
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare handle-event)
+
+(defonce enabled-mattermost (atom true))
+
+(s/def ::uri ::us/string)
+
+(defmethod ig/pre-init-spec ::reporter [_]
+  (s/keys :req-un [::wrk/executor ::db/pool ::receiver]
+          :opt-un [::uri]))
+
+(defmethod ig/init-key ::reporter
+  [_ {:keys [receiver uri] :as cfg}]
+  (l/info :msg "intializing mattermost error reporter" :uri uri)
+  (let [output (a/chan (a/sliding-buffer 128)
+                       (filter #(= (:level %) "error")))]
+    (receiver :sub output)
+    (a/go-loop []
+      (let [msg (a/<! output)]
+        (if (nil? msg)
+          (l/info :msg "stoping error reporting loop")
+          (do
+            (a/<! (handle-event cfg msg))
+            (recur)))))
+    output))
+
+(defmethod ig/halt-key! ::reporter
+  [_ output]
+  (a/close! output))
+
+(defn- send-mattermost-notification!
+  [cfg {:keys [host version id] :as cdata}]
+  (try
+    (let [uri  (:uri cfg)
+          text (str "Unhandled exception:\n"
+                    "- detail: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
+                    "- profile-id: `" (:profile-id cdata) "`\n"
+                    "- host: `" host "`\n"
+                    "- version: `" version "`\n")
+          rsp    (http/send! {:uri uri
+                              :method :post
+                              :headers {"content-type" "application/json"}
+                              :body (json/encode-str {:text text})})]
+      (when (not= (:status rsp) 200)
+        (l/error :hint "error on sending data to mattermost"
+                 :response (pr-str rsp))))
+
+    (catch Exception e
+      (l/error :hint "unexpected exception on error reporter"
+               :cause e))))
+
+(defn- persist-on-database!
+  [{:keys [pool] :as cfg} {:keys [id] :as cdata}]
+  (db/with-atomic [conn pool]
+    (db/insert! conn :server-error-report
+                {:id id :content (db/tjson cdata)})))
+
+(defn- parse-context
+  [event]
+  (reduce-kv
+   (fn [acc k v]
+     (cond
+       (= k :id)         (assoc acc k (uuid/uuid v))
+       (= k :profile-id) (assoc acc k (uuid/uuid v))
+       (str/blank? v)    acc
+       :else             (assoc acc k v)))
+   {:id (uuid/next)}
+   (:context event)))
+
+(defn- parse-event
+  [event]
+  (-> (parse-context event)
+      (merge (dissoc event :context))
+      (assoc :tenant (cfg/get :tenant))
+      (assoc :host (cfg/get :host))
+      (assoc :public-uri (cfg/get :public-uri))
+      (assoc :version (:full cfg/version))))
+
+(defn handle-event
+  [{:keys [executor] :as cfg} event]
+  (aa/with-thread executor
+    (try
+      (let [cdata (parse-event event)]
+        (when (and (:uri cfg) @enabled-mattermost)
+          (send-mattermost-notification! cfg cdata))
+        (persist-on-database! cfg cdata))
+      (catch Exception e
+        (l/error :hint "unexpected exception on error reporter"
+                 :cause e)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Http Handler
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmethod ig/pre-init-spec ::handler [_]
+  (s/keys :req-un [::db/pool]))
+
+(defmethod ig/init-key ::handler
+  [_ {:keys [pool] :as cfg}]
+  (letfn [(parse-id [request]
+            (let [id (get-in request [:path-params :id])
+                  id (us/uuid-conformer id)]
+              (when (uuid? id)
+                id)))
+          (retrieve-report [id]
+            (ex/ignoring
+             (when-let [{:keys [content] :as row} (db/get-by-id pool :server-error-report id)]
+               (assoc row :content (db/decode-transit-pgobject content)))))
+
+          (render-template [{:keys [content] :as report}]
+            (some-> (io/resource "error-report.tmpl")
+                    (tmpl/render content)))]
+
+
+    (fn [request]
+      (let [result (some-> (parse-id request)
+                           (retrieve-report)
+                           (render-template))]
+        (if result
+          {:status 200
+           :headers {"content-type" "text/html; charset=utf-8"}
+           :body result}
+          {:status 404
+           :body "not found"})))))

backend/src/app/loggers/zmq.clj

@@ -0,0 +1,89 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) UXBOX Labs SL
+
+(ns app.loggers.zmq
+  "A generic ZMQ listener."
+  (:require
+   [app.common.data :as d]
+   [app.common.spec :as us]
+   [app.util.json :as json]
+   [app.util.logging :as l]
+   [app.util.time :as dt]
+   [clojure.core.async :as a]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [integrant.core :as ig])
+  (:import
+   org.zeromq.SocketType
+   org.zeromq.ZMQ$Socket
+   org.zeromq.ZContext))
+
+(declare prepare)
+(declare start-rcv-loop)
+
+(s/def ::endpoint ::us/string)
+
+(defmethod ig/pre-init-spec ::receiver [_]
+  (s/keys :opt-un [::endpoint]))
+
+(defmethod ig/init-key ::receiver
+  [_ {:keys [endpoint] :as cfg}]
+  (l/info :msg "intializing ZMQ receiver" :bind endpoint)
+  (let [buffer (a/chan 1)
+        output (a/chan 1 (comp (filter map?)
+                               (map prepare)))
+        mult   (a/mult output)]
+    (when endpoint
+      (a/thread (start-rcv-loop {:out buffer :endpoint endpoint})))
+    (a/pipe buffer output)
+    (with-meta
+      (fn [cmd ch]
+        (case cmd
+          :sub (a/tap mult ch)
+          :unsub (a/untap mult ch))
+        ch)
+      {::output output
+       ::buffer buffer
+       ::mult mult})))
+
+(defmethod ig/halt-key! ::receiver
+  [_ f]
+  (a/close! (::buffer (meta f))))
+
+(defn- start-rcv-loop
+  ([] (start-rcv-loop nil))
+  ([{:keys [out endpoint] :or {endpoint "tcp://localhost:5556"}}]
+   (let [out    (or out (a/chan 1))
+         zctx   (ZContext.)
+         socket (.. zctx (createSocket SocketType/SUB))]
+     (.. socket (connect ^String endpoint))
+     (.. socket (subscribe ""))
+     (.. socket (setReceiveTimeOut 5000))
+     (loop []
+       (let [msg (.recv ^ZMQ$Socket socket)
+             msg (json/decode msg)
+             msg (if (nil? msg) :empty msg)]
+         (if (a/>!! out msg)
+           (recur)
+           (do
+             (.close ^java.lang.AutoCloseable socket)
+             (.close ^java.lang.AutoCloseable zctx))))))))
+
+(defn- prepare
+  [event]
+  (d/merge
+   {:logger     (:loggerName event)
+    :level      (str/lower (:level event))
+    :thread     (:thread event)
+    :created-at (dt/instant (:timeMillis event))
+    :message    (:message event)}
+   (when-let [ctx (:contextMap event)]
+     {:context ctx})
+   (when-let [thrown (:thrown event)]
+     {:error
+      {:class (:name thrown)
+       :message (:message thrown)
+       :trace (:extendedStackTrace thrown)}})))

backend/src/app/main.clj

@@ -2,41 +2,339 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.main
   (:require
-   [app.config :as cfg]
-   [clojure.tools.logging :as log]
-   [mount.core :as mount]))
+   [app.config :as cf]
+   [app.util.logging :as l]
+   [app.util.time :as dt]
+   [integrant.core :as ig]))
 
-(defn- enable-asserts
-  [_]
-  (let [m (System/getProperty "app.enable-asserts")]
-    (or (nil? m) (= "true" m))))
+(def system-config
+  {:app.db/pool
+   {:uri        (cf/get :database-uri)
+    :username   (cf/get :database-username)
+    :password   (cf/get :database-password)
+    :metrics    (ig/ref :app.metrics/metrics)
+    :migrations (ig/ref :app.migrations/all)
+    :name :main
+    :min-pool-size 0
+    :max-pool-size 20}
 
-;; Set value for all new threads bindings.
-(alter-var-root #'*assert* enable-asserts)
+   :app.metrics/metrics
+   {:definitions
+    {:profile-register
+     {:name "actions_profile_register_count"
+      :help "A global counter of user registrations."
+      :type :counter}
+     :profile-activation
+     {:name "actions_profile_activation_count"
+      :help "A global counter of profile activations"
+      :type :counter}}}
 
-;; Set value for current thread binding.
-(set! *assert* (enable-asserts nil))
+   :app.migrations/all
+   {:main (ig/ref :app.migrations/migrations)}
 
-;; --- Entry point
+   :app.migrations/migrations
+   {}
 
-(defn run
-  [_params]
-  (require 'app.srepl.server
-           'app.services
-           'app.migrations
-           'app.worker
-           'app.media
-           'app.http)
-  (mount/start)
-  (log/infof "Welcome to penpot! Version: '%s'." (:full @cfg/version)))
+   :app.msgbus/msgbus
+   {:backend   (cf/get :msgbus-backend :redis)
+    :redis-uri (cf/get :redis-uri)}
+
+   :app.tokens/tokens
+   {:props (ig/ref :app.setup/props)}
+
+   :app.storage/gc-deleted-task
+   {:pool     (ig/ref :app.db/pool)
+    :storage  (ig/ref :app.storage/storage)
+    :min-age  (dt/duration {:hours 2})}
+
+   :app.storage/gc-touched-task
+   {:pool     (ig/ref :app.db/pool)}
+
+   :app.storage/recheck-task
+   {:pool     (ig/ref :app.db/pool)
+    :storage  (ig/ref :app.storage/storage)}
+
+   :app.http.session/session
+   {:pool   (ig/ref :app.db/pool)
+    :tokens (ig/ref :app.tokens/tokens)}
+
+   :app.http.session/gc-task
+   {:pool        (ig/ref :app.db/pool)
+    :max-age     (cf/get :http-session-idle-max-age)}
+
+   :app.http.session/updater
+   {:pool           (ig/ref :app.db/pool)
+    :metrics        (ig/ref :app.metrics/metrics)
+    :executor       (ig/ref :app.worker/executor)
+    :session        (ig/ref :app.http.session/session)
+    :max-batch-age  (cf/get :http-session-updater-batch-max-age)
+    :max-batch-size (cf/get :http-session-updater-batch-max-size)}
+
+   :app.http.awsns/handler
+   {:tokens  (ig/ref :app.tokens/tokens)
+    :pool    (ig/ref :app.db/pool)}
+
+   :app.http/server
+   {:port    (cf/get :http-server-port)
+    :router  (ig/ref :app.http/router)
+    :metrics (ig/ref :app.metrics/metrics)
+    :ws      {"/ws/notifications" (ig/ref :app.notifications/handler)}}
+
+   :app.http/router
+   {:rpc         (ig/ref :app.rpc/rpc)
+    :session     (ig/ref :app.http.session/session)
+    :tokens      (ig/ref :app.tokens/tokens)
+    :public-uri  (cf/get :public-uri)
+    :metrics     (ig/ref :app.metrics/metrics)
+    :oauth       (ig/ref :app.http.oauth/handlers)
+    :assets      (ig/ref :app.http.assets/handlers)
+    :storage     (ig/ref :app.storage/storage)
+    :sns-webhook (ig/ref :app.http.awsns/handler)
+    :feedback    (ig/ref :app.http.feedback/handler)
+    :error-report-handler (ig/ref :app.loggers.mattermost/handler)}
+
+   :app.http.assets/handlers
+   {:metrics           (ig/ref :app.metrics/metrics)
+    :assets-path       (cf/get :assets-path)
+    :storage           (ig/ref :app.storage/storage)
+    :cache-max-age     (dt/duration {:hours 24})
+    :signature-max-age (dt/duration {:hours 24 :minutes 5})}
+
+   :app.http.feedback/handler
+   {:pool (ig/ref :app.db/pool)}
+
+   :app.http.oauth/handlers
+   {:rpc           (ig/ref :app.rpc/rpc)
+    :session       (ig/ref :app.http.session/session)
+    :tokens        (ig/ref :app.tokens/tokens)
+    :public-uri    (cf/get :public-uri)}
+
+   ;; RLimit definition for password hashing
+   :app.rlimits/password
+   (cf/get :rlimits-password)
+
+   ;; RLimit definition for image processing
+   :app.rlimits/image
+   (cf/get :rlimits-image)
+
+   ;; RLimit definition for font processing
+   :app.rlimits/font
+   (cf/get :rlimits-font 2)
+
+   ;; A collection of rlimits as hash-map.
+   :app.rlimits/all
+   {:password (ig/ref :app.rlimits/password)
+    :image    (ig/ref :app.rlimits/image)
+    :font     (ig/ref :app.rlimits/font)}
+
+   :app.rpc/rpc
+   {:pool       (ig/ref :app.db/pool)
+    :session    (ig/ref :app.http.session/session)
+    :tokens     (ig/ref :app.tokens/tokens)
+    :metrics    (ig/ref :app.metrics/metrics)
+    :storage    (ig/ref :app.storage/storage)
+    :msgbus     (ig/ref :app.msgbus/msgbus)
+    :rlimits    (ig/ref :app.rlimits/all)
+    :public-uri (cf/get :public-uri)
+    :audit      (ig/ref :app.loggers.audit/collector)}
+
+   :app.notifications/handler
+   {:msgbus   (ig/ref :app.msgbus/msgbus)
+    :pool     (ig/ref :app.db/pool)
+    :session  (ig/ref :app.http.session/session)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :executor (ig/ref :app.worker/executor)}
+
+   :app.worker/executor
+   {:min-threads 0
+    :max-threads 256
+    :idle-timeout 60000
+    :name :worker}
+
+   :app.worker/worker
+   {:executor (ig/ref :app.worker/executor)
+    :tasks    (ig/ref :app.worker/registry)
+    :metrics  (ig/ref :app.metrics/metrics)
+    :pool     (ig/ref :app.db/pool)}
+
+   :app.worker/scheduler
+   {:executor   (ig/ref :app.worker/executor)
+    :tasks      (ig/ref :app.worker/registry)
+    :pool       (ig/ref :app.db/pool)
+    :schedule
+    [{:cron #app/cron "0 0 0 * * ? *" ;; daily
+      :task :file-media-gc}
+
+     {:cron #app/cron "0 0 * * * ?"  ;; hourly
+      :task :file-xlog-gc}
+
+     {:cron #app/cron "0 0 1 * * ?"  ;; daily (1 hour shift)
+      :task :storage-deleted-gc}
+
+     {:cron #app/cron "0 0 2 * * ?"  ;; daily (2 hour shift)
+      :task :storage-touched-gc}
+
+     {:cron #app/cron "0 0 3 * * ?"  ;; daily (3 hour shift)
+      :task :session-gc}
+
+     {:cron #app/cron "0 0 * * * ?"  ;; hourly
+      :task :storage-recheck}
+
+     {:cron #app/cron "0 0 0 * * ?"  ;; daily
+      :task :tasks-gc}
+
+     (when (cf/get :audit-archive-enabled)
+       {:cron #app/cron "0 0 * * * ?" ;; every 1h
+        :task :audit-archive})
+
+     (when (cf/get :audit-archive-gc-enabled)
+       {:cron #app/cron "0 0 * * * ?" ;; every 1h
+        :task :audit-archive-gc})
+
+     (when (cf/get :telemetry-enabled)
+       {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
+        :task :telemetry})]}
+
+   :app.worker/registry
+   {:metrics (ig/ref :app.metrics/metrics)
+    :tasks
+    {:sendmail           (ig/ref :app.emails/sendmail-handler)
+     :delete-object      (ig/ref :app.tasks.delete-object/handler)
+     :delete-profile     (ig/ref :app.tasks.delete-profile/handler)
+     :file-media-gc      (ig/ref :app.tasks.file-media-gc/handler)
+     :file-xlog-gc       (ig/ref :app.tasks.file-xlog-gc/handler)
+     :storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
+     :storage-touched-gc (ig/ref :app.storage/gc-touched-task)
+     :storage-recheck    (ig/ref :app.storage/recheck-task)
+     :tasks-gc           (ig/ref :app.tasks.tasks-gc/handler)
+     :telemetry          (ig/ref :app.tasks.telemetry/handler)
+     :session-gc         (ig/ref :app.http.session/gc-task)
+     :audit-archive      (ig/ref :app.loggers.audit/archive-task)
+     :audit-archive-gc   (ig/ref :app.loggers.audit/archive-gc-task)}}
+
+   :app.emails/sendmail-handler
+   {:host             (cf/get :smtp-host)
+    :port             (cf/get :smtp-port)
+    :ssl              (cf/get :smtp-ssl)
+    :tls              (cf/get :smtp-tls)
+    :enabled          (cf/get :smtp-enabled)
+    :username         (cf/get :smtp-username)
+    :password         (cf/get :smtp-password)
+    :metrics          (ig/ref :app.metrics/metrics)
+    :default-reply-to (cf/get :smtp-default-reply-to)
+    :default-from     (cf/get :smtp-default-from)}
+
+   :app.tasks.tasks-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age cf/deletion-delay}
+
+   :app.tasks.delete-object/handler
+   {:pool    (ig/ref :app.db/pool)
+    :storage (ig/ref :app.storage/storage)}
+
+   :app.tasks.delete-profile/handler
+   {:pool    (ig/ref :app.db/pool)}
+
+   :app.tasks.file-media-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age cf/deletion-delay}
+
+   :app.tasks.file-xlog-gc/handler
+   {:pool    (ig/ref :app.db/pool)
+    :max-age (dt/duration {:hours 24})}
+
+   :app.tasks.telemetry/handler
+   {:pool        (ig/ref :app.db/pool)
+    :version     (:full cf/version)
+    :uri         (cf/get :telemetry-uri)
+    :sprops      (ig/ref :app.setup/props)}
+
+   :app.srepl/server
+   {:port (cf/get :srepl-port)
+    :host (cf/get :srepl-host)}
+
+   :app.setup/props
+   {:pool (ig/ref :app.db/pool)
+    :key  (cf/get :secret-key)}
+
+   :app.loggers.zmq/receiver
+   {:endpoint (cf/get :loggers-zmq-uri)}
+
+   :app.loggers.audit/collector
+   {:enabled  (cf/get :audit-enabled false)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
+
+   :app.loggers.audit/archive-task
+   {:uri      (cf/get :audit-archive-uri)
+    :enabled  (cf/get :audit-archive-enabled false)
+    :tokens   (ig/ref :app.tokens/tokens)
+    :pool     (ig/ref :app.db/pool)}
+
+   :app.loggers.audit/archive-gc-task
+   {:enabled  (cf/get :audit-archive-gc-enabled false)
+    :max-age  (cf/get :audit-archive-gc-max-age cf/deletion-delay)
+    :pool     (ig/ref :app.db/pool)}
+
+   :app.loggers.loki/reporter
+   {:uri      (cf/get :loggers-loki-uri)
+    :receiver (ig/ref :app.loggers.zmq/receiver)
+    :executor (ig/ref :app.worker/executor)}
+
+   :app.loggers.mattermost/reporter
+   {:uri      (cf/get :error-report-webhook)
+    :receiver (ig/ref :app.loggers.zmq/receiver)
+    :pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)}
+
+   :app.loggers.mattermost/handler
+   {:pool (ig/ref :app.db/pool)}
+
+   :app.storage/storage
+   {:pool     (ig/ref :app.db/pool)
+    :executor (ig/ref :app.worker/executor)
+    :backend  (cf/get :storage-backend :fs)
+    :backends {:s3  (ig/ref [::main :app.storage.s3/backend])
+               :db  (ig/ref [::main :app.storage.db/backend])
+               :fs  (ig/ref [::main :app.storage.fs/backend])
+               :tmp (ig/ref [::tmp  :app.storage.fs/backend])}}
+
+   [::main :app.storage.s3/backend]
+   {:region (cf/get :storage-s3-region)
+    :bucket (cf/get :storage-s3-bucket)}
+
+   [::main :app.storage.fs/backend]
+   {:directory (cf/get :storage-fs-directory)}
+
+   [::tmp :app.storage.fs/backend]
+   {:directory "/tmp/penpot"}
+
+   [::main :app.storage.db/backend]
+   {:pool (ig/ref :app.db/pool)}})
+
+(def system nil)
+
+(defn start
+  []
+  (ig/load-namespaces system-config)
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             (-> system-config
+                                 (ig/prep)
+                                 (ig/init))))
+  (l/info :msg "welcome to penpot"
+          :version (:full cf/version)))
+
+(defn stop
+  []
+  (alter-var-root #'system (fn [sys]
+                             (when sys (ig/halt! sys))
+                             nil)))
 
 (defn -main
   [& _args]
-  (run {}))
+  (start))

backend/src/app/media.clj

@@ -2,42 +2,42 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.media
-  "Media postprocessing."
+  "Media & Font postprocessing."
   (:require
+   [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.media :as cm]
    [app.common.spec :as us]
-   [app.config :as cfg]
-   [app.util.http :as http]
-   [clojure.core.async :as a]
+   [app.rlimits :as rlm]
+   [app.rpc.queries.svg :as svg]
+   [buddy.core.bytes :as bb]
+   [buddy.core.codecs :as bc]
    [clojure.java.io :as io]
+   [clojure.java.shell :as sh]
    [clojure.spec.alpha :as s]
-   [datoteka.core :as fs]
-   [mount.core :refer [defstate]])
+   [cuerdas.core :as str]
+   [datoteka.core :as fs])
   (:import
    java.io.ByteArrayInputStream
-   java.util.concurrent.Semaphore
+   java.io.OutputStream
+   org.apache.commons.io.IOUtils
    org.im4java.core.ConvertCmd
    org.im4java.core.IMOperation
    org.im4java.core.Info))
 
-(declare semaphore)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; --- Utility functions
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defstate semaphore
-  :start (Semaphore. (:image-process-max-threads cfg/config 1)))
-
-
-;; --- Generic specs
+(s/def ::image-content-type cm/valid-image-types)
+(s/def ::font-content-type cm/valid-font-types)
 
 (s/def :internal.http.upload/filename ::us/string)
 (s/def :internal.http.upload/size ::us/integer)
-(s/def :internal.http.upload/content-type cm/valid-media-types)
+(s/def :internal.http.upload/content-type ::us/string)
 (s/def :internal.http.upload/tempfile any?)
 
 (s/def ::upload
@@ -46,8 +46,44 @@
                    :internal.http.upload/tempfile
                    :internal.http.upload/content-type]))
 
+(defn validate-media-type
+  ([mtype] (validate-media-type mtype cm/valid-image-types))
+  ([mtype allowed]
+   (when-not (contains? allowed mtype)
+     (ex/raise :type :validation
+               :code :media-type-not-allowed
+               :hint "Seems like you are uploading an invalid media object"))))
 
+
+(defmulti process :cmd)
+(defmulti process-error class)
+
+(defmethod process :default
+  [{:keys [cmd] :as params}]
+  (ex/raise :type :internal
+            :code :not-implemented
+            :hint (str/fmt "No impl found for process cmd: %s" cmd)))
+
+(defmethod process-error :default
+  [error]
+  (throw error))
+
+(defn run
+  [{:keys [rlimits] :as cfg} {:keys [rlimit] :or {rlimit :image} :as params}]
+  (us/assert map? rlimits)
+  (let [rlimit (get rlimits rlimit)]
+    (when-not rlimit
+      (ex/raise :type :internal
+                :code :rlimit-not-configured
+                :hint ":image rlimit not configured"))
+    (try
+      (rlm/execute rlimit (process params))
+      (catch Throwable e
+        (process-error e)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; --- Thumbnails Generation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (s/def ::cmd keyword?)
 
@@ -75,7 +111,7 @@
   (let [{:keys [path mtype]} input
         format (or (cm/mtype->format mtype) format)
         ext    (cm/format->extension format)
-        tmp (fs/create-tempfile :suffix ext)]
+        tmp    (fs/create-tempfile :suffix ext)]
 
     (doto (ConvertCmd.)
       (.run operation (into-array (map str [path tmp]))))
@@ -85,10 +121,9 @@
       (assoc params
              :format format
              :mtype  (cm/format->mtype format)
+             :size   (alength ^bytes thumbnail-data)
              :data   (ByteArrayInputStream. thumbnail-data)))))
 
-(defmulti process :cmd)
-
 (defmethod process :generic-thumbnail
   [{:keys [quality width height] :as params}]
   (us/assert ::thumbnail-params params)
@@ -96,7 +131,7 @@
              (.addImage)
              (.autoOrient)
              (.strip)
-             (.thumbnail (int width) (int height) ">")
+             (.thumbnail ^Integer (int width) ^Integer (int height) ">")
              (.quality (double quality))
              (.addImage))]
     (generic-process (assoc params :operation op))))
@@ -108,85 +143,190 @@
              (.addImage)
              (.autoOrient)
              (.strip)
-             (.thumbnail (int width) (int height) "^")
+             (.thumbnail ^Integer (int width) ^Integer (int height) "^")
              (.gravity "center")
              (.extent (int width) (int height))
              (.quality (double quality))
              (.addImage))]
     (generic-process (assoc params :operation op))))
 
+(defn get-basic-info-from-svg
+  [{:keys [tag attrs] :as data}]
+  (when (not= tag :svg)
+    (ex/raise :type :validation
+              :code :unable-to-parse-svg
+              :hint "uploaded svg has invalid content"))
+  (reduce (fn [default f]
+            (if-let [res (f attrs)]
+              (reduced res)
+              default))
+          {:width 100 :height 100}
+          [(fn parse-width-and-height
+             [{:keys [width height]}]
+             (when (and (string? width)
+                        (string? height))
+               (let [width  (d/parse-double width)
+                     height (d/parse-double height)]
+                 (when (and width height)
+                   {:width (int width)
+                    :height (int height)}))))
+           (fn parse-viewbox
+             [{:keys [viewBox]}]
+             (let [[x y width height] (->> (str/split viewBox #"\s+" 4)
+                                           (map d/parse-double))]
+               (when (and x y width height)
+                 {:width (int width)
+                  :height (int height)})))]))
+
 (defmethod process :info
   [{:keys [input] :as params}]
   (us/assert ::input input)
   (let [{:keys [path mtype]} input]
     (if (= mtype "image/svg+xml")
-      {:width 100
-       :height 100
-       :mtype mtype}
+      (let [info (some-> path slurp svg/parse get-basic-info-from-svg)]
+        (when-not info
+          (ex/raise :type :validation
+                    :code :invalid-svg-file
+                    :hint "uploaded svg does not provides dimensions"))
+        (assoc info :mtype mtype))
+
       (let [instance (Info. (str path))
             mtype'   (.getProperty instance "Mime type")]
         (when (and (string? mtype)
                    (not= mtype mtype'))
           (ex/raise :type :validation
                     :code :media-type-mismatch
-                    :hint "Seems like you are uploading a file whose content does not match the extension."))
-        {:width  (.getImageWidth instance)
-         :height (.getImageHeight instance)
-         :mtype  mtype'}))))
+                    :hint (str "Seems like you are uploading a file whose content does not match the extension."
+                               "Expected: " mtype ". Got: " mtype')))
+        ;; For an animated GIF, getImageWidth/Height returns the delta size of one frame (if no frame given
+        ;; it returns size of the last one), whereas getPageWidth/Height always return the full size of
+        ;; any frame.
+        {:width  (.getPageWidth instance)
+         :height (.getPageHeight instance)
+         :mtype  mtype}))))
 
-(defmethod process :default
-  [{:keys [cmd] :as params}]
-  (ex/raise :type :internal
-            :code :not-implemented
-            :hint (str "No impl found for process cmd:" cmd)))
+(defmethod process-error org.im4java.core.InfoException
+  [error]
+  (ex/raise :type :validation
+            :code :invalid-image
+            :hint "invalid image"
+            :cause error))
 
-(defn run
-  [params]
-  (try
-    (.acquire semaphore)
-    (let [res (a/<!! (a/thread
-                       (try
-                         (process params)
-                         (catch Throwable e
-                           e))))]
-      (if (instance? Throwable res)
-        (throw res)
-        res))
-    (finally
-      (.release semaphore))))
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; --- Fonts Generation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def all-fotmats #{"font/woff2", "font/woff", "font/otf", "font/ttf"})
+
+(defmethod process :generate-fonts
+  [{:keys [input] :as params}]
+  (letfn [(ttf->otf [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot")
+                  output-file (fs/path (str input-file ".otf"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "fontforge" "-lang=ff" "-c"
+                                     (str/fmt "Open('%s'); Generate('%s')"
+                                              (str input-file)
+                                              (str output-file)))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
 
 
-;; --- Utility functions
+          (otf->ttf [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot")
+                  output-file (fs/path (str input-file ".ttf"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "fontforge" "-lang=ff" "-c"
+                                     (str/fmt "Open('%s'); Generate('%s')"
+                                              (str input-file)
+                                              (str output-file)))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
 
-(defn validate-media-type
-  [media-type]
-  (when-not (cm/valid-media-types media-type)
-    (ex/raise :type :validation
-              :code :media-type-not-allowed
-              :hint "Seems like you are uploading an invalid media object")))
+          (ttf-or-otf->woff [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  output-file (fs/path (str input-file ".woff"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "sfnt2woff" (str input-file))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
 
+          (ttf-or-otf->woff2 [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  output-file (fs/path (str input-file ".woff2"))
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "woff2_compress" (str input-file))]
+              (when (zero? (:exit res))
+                (fs/slurp-bytes output-file))))
 
-;; TODO: rewrite using jetty http client instead of jvm
-;; builtin (because builtin http client uses a lot of memory for the
-;; same operation.
+          (woff->sfnt [data]
+            (let [input-file  (fs/create-tempfile :prefix "penpot" :suffix "")
+                  _           (with-open [out (io/output-stream input-file)]
+                                (IOUtils/writeChunked ^bytes data ^OutputStream out)
+                                (.flush ^OutputStream out))
+                  res         (sh/sh "woff2sfnt" (str input-file)
+                                     :out-enc :bytes)]
+              (when (zero? (:exit res))
+                (:out res))))
 
-(defn download-media-object
-  [url]
-  (let [result (http/get! url {:as :byte-array})
-        data (:body result)
-        content-type (get (:headers result) "content-type")
-        format (cm/mtype->format content-type)]
-    (if (nil? format)
-      (ex/raise :type :validation
-                :code :media-type-not-allowed
-                :hint "Seems like the url points to an invalid media object.")
-      (let [tempfile (fs/create-tempfile)
-            base-filename (first (fs/split-ext (fs/name tempfile)))
-            filename (str base-filename (cm/format->extension format))]
-        (with-open [ostream (io/output-stream tempfile)]
-          (.write ostream data))
-        {:filename filename
-         :size (count data)
-         :tempfile tempfile
-         :content-type content-type}))))
+          ;; Documented here:
+          ;; https://docs.microsoft.com/en-us/typography/opentype/spec/otff#table-directory
+          (get-sfnt-type [data]
+            (let [buff (bb/slice data 0 4)
+                  type (bc/bytes->hex buff)]
+              (case type
+                "4f54544f" :otf
+                "00010000" :ttf
+                (ex/raise :type :internal
+                          :code :unexpected-data
+                          :hint "unexpected font data"))))
 
+          (gen-if-nil [val factory]
+            (if (nil? val)
+              (factory)
+              val))]
+
+    (let [current (into #{} (keys input))]
+      (cond
+        (contains? current "font/ttf")
+        (let [data (get input "font/ttf")]
+          (-> input
+              (update "font/otf" gen-if-nil #(ttf->otf data))
+              (update "font/woff" gen-if-nil #(ttf-or-otf->woff data))
+              (assoc "font/woff2" (ttf-or-otf->woff2 data))))
+
+        (contains? current "font/otf")
+        (let [data (get input "font/otf")]
+          (-> input
+              (update "font/woff" gen-if-nil #(ttf-or-otf->woff data))
+              (assoc "font/ttf" (otf->ttf data))
+              (assoc "font/woff2" (ttf-or-otf->woff2 data))))
+
+        (contains? current "font/woff")
+        (let [data (get input "font/woff")
+              sfnt (woff->sfnt data)]
+          (when-not sfnt
+            (ex/raise :type :validation
+                      :code :invalid-woff-file
+                      :hint "invalid woff file"))
+          (let [stype (get-sfnt-type sfnt)]
+            (cond-> input
+              true
+              (-> (assoc "font/woff" data)
+                  (assoc "font/woff2" (ttf-or-otf->woff2 sfnt)))
+
+              (= stype :otf)
+              (-> (assoc "font/otf" sfnt)
+                  (assoc "font/ttf" (otf->ttf sfnt)))
+
+              (= stype :ttf)
+              (-> (assoc "font/otf" (ttf->otf sfnt))
+                  (assoc "font/ttf" sfnt)))))))))

backend/src/app/media_storage.clj

@@ -1,37 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
-
-(ns app.media-storage
-  "A media storage impl for app."
-  (:require
-   [app.config :refer [config]]
-   [app.util.storage :as ust]
-   [mount.core :refer [defstate]]))
-
-;; --- State
-
-(declare assets-storage)
-
-(defstate assets-storage
-  :start (ust/create {:base-path (:assets-directory config)
-                      :base-uri (:assets-uri config)}))
-
-(declare media-storage)
-
-(defstate media-storage
-  :start (ust/create {:base-path (:media-directory config)
-                      :base-uri (:media-uri config)
-                      :xf (comp ust/random-path
-                                ust/slugify-filename)}))
-
-;; --- Public Api
-
-(defn resolve-asset
-  [path]
-  (str (ust/public-uri assets-storage path)))

backend/src/app/metrics.clj

@@ -2,44 +2,95 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.metrics
+  (:require
+   [app.common.exceptions :as ex]
+   [app.util.logging :as l]
+   [clojure.spec.alpha :as s]
+   [integrant.core :as ig])
   (:import
    io.prometheus.client.CollectorRegistry
    io.prometheus.client.Counter
    io.prometheus.client.Gauge
    io.prometheus.client.Summary
+   io.prometheus.client.Histogram
    io.prometheus.client.exporter.common.TextFormat
    io.prometheus.client.hotspot.DefaultExports
+   io.prometheus.client.jetty.JettyStatisticsCollector
+   org.eclipse.jetty.server.handler.StatisticsHandler
    java.io.StringWriter))
 
-(defn- create-registry
+(declare instrument-vars!)
+(declare instrument)
+(declare create-registry)
+(declare create)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Entry Point
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn- handler
+  [registry _request]
+  (let [samples  (.metricFamilySamples ^CollectorRegistry registry)
+        writer   (StringWriter.)]
+    (TextFormat/write004 writer samples)
+    {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
+     :body (.toString writer)}))
+
+(s/def ::definitions
+  (s/map-of keyword? map?))
+
+(defmethod ig/pre-init-spec ::metrics [_]
+  (s/keys :opt-un [::definitions]))
+
+(defmethod ig/init-key ::metrics
+  [_ {:keys [definitions] :as cfg}]
+  (l/info :action "initialize metrics")
+  (let [registry    (create-registry)
+        definitions (reduce-kv (fn [res k v]
+                                 (->> (assoc v :registry registry)
+                                      (create)
+                                      (assoc res k)))
+                               {}
+                               definitions)]
+    {:handler (partial handler registry)
+     :definitions definitions
+     :registry registry}))
+
+(s/def ::handler fn?)
+(s/def ::registry #(instance? CollectorRegistry %))
+(s/def ::metrics
+  (s/keys :req-un [::registry ::handler]))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Implementation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn create-registry
   []
   (let [registry (CollectorRegistry.)]
     (DefaultExports/register registry)
     registry))
 
-(defonce registry (create-registry))
-(defonce cache (atom {}))
-
 (defmacro with-measure
-  [sym expr teardown]
-  `(let [~sym (System/nanoTime)]
+  [& {:keys [expr cb]}]
+  `(let [start# (System/nanoTime)
+         tdown# ~cb]
      (try
        ~expr
        (finally
-         (let [~sym (/ (- (System/nanoTime) ~sym) 1000000)]
-           ~teardown)))))
+         (tdown# (/ (- (System/nanoTime) start#) 1000000))))))
 
 (defn make-counter
-  [{:keys [id help] :as props}]
-  (let [instance (doto (Counter/build)
-                   (.name id)
-                   (.help help))
+  [{:keys [name help registry reg labels] :as props}]
+  (let [registry (or registry reg)
+        instance (.. (Counter/build)
+                     (name name)
+                     (help help))
+        _        (when (seq labels)
+                   (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
     (reify
       clojure.lang.IDeref
@@ -49,34 +100,21 @@
       (invoke [_ cmd]
         (.inc ^Counter instance))
 
-      (invoke [_ cmd val]
-        (case cmd
-          :wrap (fn
-                  ([a]
-                   (.inc ^Counter instance)
-                   (val a))
-                  ([a b]
-                   (.inc ^Counter instance)
-                   (val a b))
-                  ([a b c]
-                   (.inc ^Counter instance)
-                   (val a b c)))
-
-          (throw (IllegalArgumentException. "invalid arguments")))))))
-
-(defn counter
-  [{:keys [id] :as props}]
-  (or (get @cache id)
-      (let [v (make-counter props)]
-        (swap! cache assoc id v)
-        v)))
+      (invoke [_ cmd labels]
+        (.. ^Counter instance
+            (labels (into-array String labels))
+            (inc))))))
 
 (defn make-gauge
-  [{:keys [id help] :as props}]
-  (let [instance (doto (Gauge/build)
-                   (.name id)
-                   (.help help))
+  [{:keys [name help registry reg labels] :as props}]
+  (let [registry (or registry reg)
+        instance (.. (Gauge/build)
+                     (name name)
+                     (help help))
+        _        (when (seq labels)
+                   (.labelNames instance (into-array String labels)))
         instance (.register instance registry)]
+
     (reify
       clojure.lang.IDeref
       (deref [_] instance)
@@ -85,94 +123,186 @@
       (invoke [_ cmd]
         (case cmd
           :inc (.inc ^Gauge instance)
-          :dec (.dec ^Gauge instance))))))
+          :dec (.dec ^Gauge instance)))
 
-(defn gauge
-  [{:keys [id] :as props}]
-  (or (get @cache id)
-      (let [v (make-gauge props)]
-        (swap! cache assoc id v)
-        v)))
+      (invoke [_ cmd labels]
+        (let [labels (into-array String [labels])]
+          (case cmd
+            :inc (.. ^Gauge instance (labels labels) (inc))
+            :dec (.. ^Gauge instance (labels labels) (dec))))))))
+
+(def default-quantiles
+  [[0.75 0.02]
+   [0.99 0.001]])
 
 (defn make-summary
-  [{:keys [id help] :as props}]
-  (let [instance (doto (Summary/build)
-                   (.name id)
-                   (.help help)
-                   (.quantile 0.5 0.05)
-                   (.quantile 0.9 0.01)
-                   (.quantile 0.99 0.001))
-        instance  (.register instance registry)]
+  [{:keys [name help registry reg labels max-age quantiles buckets]
+    :or {max-age 3600 buckets 6 quantiles default-quantiles} :as props}]
+  (let [registry (or registry reg)
+        instance (doto (Summary/build)
+                   (.name name)
+                   (.help help))
+        _        (when (seq quantiles)
+                   (.maxAgeSeconds ^Summary instance max-age)
+                   (.ageBuckets ^Summary instance buckets))
+        _        (doseq [[q e] quantiles]
+                   (.quantile ^Summary instance q e))
+        _        (when (seq labels)
+                   (.labelNames instance (into-array String labels)))
+        instance (.register instance registry)]
     (reify
       clojure.lang.IDeref
       (deref [_] instance)
 
       clojure.lang.IFn
-      (invoke [_ val]
+      (invoke [_ cmd val]
         (.observe ^Summary instance val))
 
+      (invoke [_ cmd val labels]
+        (.. ^Summary instance
+            (labels (into-array String labels))
+            (observe val))))))
+
+(def default-histogram-buckets
+  [1 5 10 25 50 75 100 250 500 750 1000 2500 5000 7500])
+
+(defn make-histogram
+  [{:keys [name help registry reg labels buckets]
+    :or {buckets default-histogram-buckets}}]
+  (let [registry (or registry reg)
+        instance (doto (Histogram/build)
+                   (.name name)
+                   (.help help)
+                   (.buckets (into-array Double/TYPE buckets)))
+        _        (when (seq labels)
+                   (.labelNames instance (into-array String labels)))
+        instance (.register instance registry)]
+    (reify
+      clojure.lang.IDeref
+      (deref [_] instance)
+
+      clojure.lang.IFn
       (invoke [_ cmd val]
-        (case cmd
-          :wrap (fn
-                  ([a]
-                   (with-measure $$
-                     (val a)
-                     (.observe ^Summary instance $$)))
-                  ([a b]
-                   (with-measure $$
-                     (val a b)
-                     (.observe ^Summary instance $$)))
-                  ([a b c]
-                   (with-measure $$
-                     (val a b c)
-                     (.observe ^Summary instance $$))))
+        (.observe ^Histogram instance val))
 
-          (throw (IllegalArgumentException. "invalid arguments")))))))
+      (invoke [_ cmd val labels]
+        (.. ^Histogram instance
+            (labels (into-array String labels))
+            (observe val))))))
 
-(defn summary
-  [{:keys [id] :as props}]
-  (or (get @cache id)
-      (let [v (make-summary props)]
-        (swap! cache assoc id v)
-        v)))
-
-(defn wrap-summary
-  [f props]
-  (let [sm (summary props)]
-    (sm :wrap f)))
+(defn create
+  [{:keys [type] :as props}]
+  (case type
+    :counter (make-counter props)
+    :gauge   (make-gauge props)
+    :summary (make-summary props)
+    :histogram (make-histogram props)))
 
 (defn wrap-counter
-  [f props]
-  (let [cnt (counter props)]
-    (cnt :wrap f)))
+  ([rootf mobj]
+   (let [mdata (meta rootf)
+         origf (::original mdata rootf)]
+     (with-meta
+       (fn
+         ([a]
+          (mobj :inc)
+          (origf a))
+         ([a b]
+          (mobj :inc)
+          (origf a b))
+         ([a b & more]
+          (mobj :inc)
+          (apply origf a b more)))
+       (assoc mdata ::original origf))))
+  ([rootf mobj labels]
+   (let [mdata  (meta rootf)
+         origf  (::original mdata rootf)]
+     (with-meta
+       (fn
+         ([a]
+          (mobj :inc labels)
+          (origf a))
+         ([a b]
+          (mobj :inc labels)
+          (origf a b))
+         ([a b & more]
+          (mobj :inc labels)
+          (apply origf a b more)))
+       (assoc mdata ::original origf)))))
 
-(defn instrument-with-counter!
-  [{:keys [var] :as props}]
-  (let [cnt  (counter props)
-        vars (if (var? var) [var] var)]
-    (doseq [var vars]
-      (alter-var-root var (fn [root]
-                            (let [mdata (meta root)
-                                  original (::counter-original mdata root)]
-                              (with-meta
-                                (cnt :wrap original)
-                                (assoc mdata ::counter-original original))))))))
+(defn wrap-summary
+  ([rootf mobj]
+   (let [mdata  (meta rootf)
+         origf  (::original mdata rootf)]
+     (with-meta
+       (fn
+         ([a]
+         (with-measure
+           :expr (origf a)
+           :cb   #(mobj :observe %)))
+         ([a b]
+          (with-measure
+            :expr (origf a b)
+            :cb   #(mobj :observe %)))
+         ([a b & more]
+          (with-measure
+            :expr (apply origf a b more)
+            :cb   #(mobj :observe %))))
+       (assoc mdata ::original origf))))
 
-(defn instrument-with-summary!
-  [{:keys [var] :as props}]
-  (let [sm (summary props)]
-    (alter-var-root var (fn [root]
-                          (let [mdata (meta root)
-                                original (::summary-original mdata root)]
-                            (with-meta
-                              (sm :wrap original)
-                              (assoc mdata ::summary-original original)))))))
+  ([rootf mobj labels]
+   (let [mdata  (meta rootf)
+         origf  (::original mdata rootf)]
+     (with-meta
+       (fn
+         ([a]
+         (with-measure
+           :expr (origf a)
+           :cb   #(mobj :observe % labels)))
+         ([a b]
+          (with-measure
+            :expr (origf a b)
+            :cb   #(mobj :observe % labels)))
+         ([a b & more]
+          (with-measure
+            :expr (apply origf a b more)
+            :cb   #(mobj :observe % labels))))
+       (assoc mdata ::original origf)))))
 
-(defn dump
-  [& _args]
-  (let [samples (.metricFamilySamples ^CollectorRegistry registry)
-        writer  (StringWriter.)]
-    (TextFormat/write004 writer samples)
-    {:headers {"content-type" TextFormat/CONTENT_TYPE_004}
-     :body (.toString writer)}))
+(defn instrument-vars!
+  [vars {:keys [wrap] :as props}]
+  (let [obj (create props)]
+    (cond
+      (instance? Counter @obj)
+      (doseq [var vars]
+        (alter-var-root var (or wrap wrap-counter) obj))
+
+      (instance? Summary @obj)
+      (doseq [var vars]
+        (alter-var-root var (or wrap wrap-summary) obj))
+
+      :else
+      (ex/raise :type :not-implemented))))
+
+(defn instrument
+  [f {:keys [wrap] :as props}]
+  (let [obj (create props)]
+    (cond
+      (instance? Counter @obj)
+      ((or wrap wrap-counter) f obj)
+
+      (instance? Summary @obj)
+      ((or wrap wrap-summary) f obj)
+
+      (instance? Histogram @obj)
+      ((or wrap wrap-summary) f obj)
+
+      :else
+      (ex/raise :type :not-implemented))))
+
+(defn instrument-jetty!
+  [^CollectorRegistry registry ^StatisticsHandler handler]
+  (doto (JettyStatisticsCollector. handler)
+    (.register registry))
+  nil)
 

backend/src/app/migrations.clj

@@ -2,133 +2,180 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.migrations
   (:require
-   [app.db :as db]
    [app.migrations.migration-0023 :as mg0023]
    [app.util.migrations :as mg]
-   [mount.core :as mount :refer [defstate]]))
+   [integrant.core :as ig]))
 
-(def +migrations+
-  {:name "uxbox-main"
-   :steps
-   [{:name "0001-add-extensions"
-     :fn (mg/resource "app/migrations/sql/0001-add-extensions.sql")}
+(def migrations
+  [{:name "0001-add-extensions"
+    :fn (mg/resource "app/migrations/sql/0001-add-extensions.sql")}
 
-    {:name "0002-add-profile-tables"
-     :fn (mg/resource "app/migrations/sql/0002-add-profile-tables.sql")}
+   {:name "0002-add-profile-tables"
+    :fn (mg/resource "app/migrations/sql/0002-add-profile-tables.sql")}
 
-    {:name "0003-add-project-tables"
-     :fn (mg/resource "app/migrations/sql/0003-add-project-tables.sql")}
+   {:name "0003-add-project-tables"
+    :fn (mg/resource "app/migrations/sql/0003-add-project-tables.sql")}
 
-    {:name "0004-add-tasks-tables"
-     :fn (mg/resource "app/migrations/sql/0004-add-tasks-tables.sql")}
+   {:name "0004-add-tasks-tables"
+    :fn (mg/resource "app/migrations/sql/0004-add-tasks-tables.sql")}
 
-    {:name "0005-add-libraries-tables"
-     :fn (mg/resource "app/migrations/sql/0005-add-libraries-tables.sql")}
+   {:name "0005-add-libraries-tables"
+    :fn (mg/resource "app/migrations/sql/0005-add-libraries-tables.sql")}
 
-    {:name "0006-add-presence-tables"
-     :fn (mg/resource "app/migrations/sql/0006-add-presence-tables.sql")}
+   {:name "0006-add-presence-tables"
+    :fn (mg/resource "app/migrations/sql/0006-add-presence-tables.sql")}
 
-    {:name "0007-drop-version-field-from-page-table"
-     :fn (mg/resource "app/migrations/sql/0007-drop-version-field-from-page-table.sql")}
+   {:name "0007-drop-version-field-from-page-table"
+    :fn (mg/resource "app/migrations/sql/0007-drop-version-field-from-page-table.sql")}
 
-    {:name "0008-add-generic-token-table"
-     :fn (mg/resource "app/migrations/sql/0008-add-generic-token-table.sql")}
+   {:name "0008-add-generic-token-table"
+    :fn (mg/resource "app/migrations/sql/0008-add-generic-token-table.sql")}
 
-    {:name "0009-drop-profile-email-table"
-     :fn (mg/resource "app/migrations/sql/0009-drop-profile-email-table.sql")}
+   {:name "0009-drop-profile-email-table"
+    :fn (mg/resource "app/migrations/sql/0009-drop-profile-email-table.sql")}
 
-    {:name "0010-add-http-session-table"
-     :fn (mg/resource "app/migrations/sql/0010-add-http-session-table.sql")}
+   {:name "0010-add-http-session-table"
+    :fn (mg/resource "app/migrations/sql/0010-add-http-session-table.sql")}
 
-    {:name "0011-add-session-id-field-to-page-change-table"
-     :fn (mg/resource "app/migrations/sql/0011-add-session-id-field-to-page-change-table.sql")}
+   {:name "0011-add-session-id-field-to-page-change-table"
+    :fn (mg/resource "app/migrations/sql/0011-add-session-id-field-to-page-change-table.sql")}
 
-    {:name "0012-make-libraries-linked-to-a-file"
-     :fn (mg/resource "app/migrations/sql/0012-make-libraries-linked-to-a-file.sql")}
+   {:name "0012-make-libraries-linked-to-a-file"
+    :fn (mg/resource "app/migrations/sql/0012-make-libraries-linked-to-a-file.sql")}
 
-    {:name "0013-mark-files-shareable"
-     :fn (mg/resource "app/migrations/sql/0013-mark-files-shareable.sql")}
+   {:name "0013-mark-files-shareable"
+    :fn (mg/resource "app/migrations/sql/0013-mark-files-shareable.sql")}
 
-    {:name "0014-refactor-media-storage.sql"
-     :fn (mg/resource "app/migrations/sql/0014-refactor-media-storage.sql")}
+   {:name "0014-refactor-media-storage.sql"
+    :fn (mg/resource "app/migrations/sql/0014-refactor-media-storage.sql")}
 
-    {:name "0015-improve-tasks-tables"
-     :fn (mg/resource "app/migrations/sql/0015-improve-tasks-tables.sql")}
+   {:name "0015-improve-tasks-tables"
+    :fn (mg/resource "app/migrations/sql/0015-improve-tasks-tables.sql")}
 
-    {:name "0016-truncate-and-alter-tokens-table"
-     :fn (mg/resource "app/migrations/sql/0016-truncate-and-alter-tokens-table.sql")}
+   {:name "0016-truncate-and-alter-tokens-table"
+    :fn (mg/resource "app/migrations/sql/0016-truncate-and-alter-tokens-table.sql")}
 
-    {:name "0017-link-files-to-libraries"
-     :fn (mg/resource "app/migrations/sql/0017-link-files-to-libraries.sql")}
+   {:name "0017-link-files-to-libraries"
+    :fn (mg/resource "app/migrations/sql/0017-link-files-to-libraries.sql")}
 
-    {:name "0018-add-file-trimming-triggers"
-     :fn (mg/resource "app/migrations/sql/0018-add-file-trimming-triggers.sql")}
+   {:name "0018-add-file-trimming-triggers"
+    :fn (mg/resource "app/migrations/sql/0018-add-file-trimming-triggers.sql")}
 
-    {:name "0019-add-improved-scheduled-tasks"
-     :fn (mg/resource "app/migrations/sql/0019-add-improved-scheduled-tasks.sql")}
+   {:name "0019-add-improved-scheduled-tasks"
+    :fn (mg/resource "app/migrations/sql/0019-add-improved-scheduled-tasks.sql")}
 
-    {:name "0020-minor-fixes-to-media-object"
-     :fn (mg/resource "app/migrations/sql/0020-minor-fixes-to-media-object.sql")}
+   {:name "0020-minor-fixes-to-media-object"
+    :fn (mg/resource "app/migrations/sql/0020-minor-fixes-to-media-object.sql")}
 
-    {:name "0021-http-session-improvements"
-     :fn (mg/resource "app/migrations/sql/0021-http-session-improvements.sql")}
+   {:name "0021-http-session-improvements"
+    :fn (mg/resource "app/migrations/sql/0021-http-session-improvements.sql")}
 
-    {:name "0022-page-file-refactor"
-     :fn (mg/resource "app/migrations/sql/0022-page-file-refactor.sql")}
+   {:name "0022-page-file-refactor"
+    :fn (mg/resource "app/migrations/sql/0022-page-file-refactor.sql")}
 
-    {:name "0023-adapt-old-pages-and-files"
-     :fn mg0023/migrate}
+   {:name "0023-adapt-old-pages-and-files"
+    :fn mg0023/migrate}
 
-    {:name "0024-mod-profile-table"
-     :fn (mg/resource "app/migrations/sql/0024-mod-profile-table.sql")}
+   {:name "0024-mod-profile-table"
+    :fn (mg/resource "app/migrations/sql/0024-mod-profile-table.sql")}
 
-    {:name "0025-del-generic-tokens-table"
-     :fn (mg/resource "app/migrations/sql/0025-del-generic-tokens-table.sql")}
+   {:name "0025-del-generic-tokens-table"
+    :fn (mg/resource "app/migrations/sql/0025-del-generic-tokens-table.sql")}
 
-    {:name "0026-mod-file-library-rel-table-synced-date"
-     :fn (mg/resource "app/migrations/sql/0026-mod-file-library-rel-table-synced-date.sql")}
+   {:name "0026-mod-file-library-rel-table-synced-date"
+    :fn (mg/resource "app/migrations/sql/0026-mod-file-library-rel-table-synced-date.sql")}
 
-    {:name "0027-mod-file-table-ignore-sync"
-     :fn (mg/resource "app/migrations/sql/0027-mod-file-table-ignore-sync.sql")}
+   {:name "0027-mod-file-table-ignore-sync"
+    :fn (mg/resource "app/migrations/sql/0027-mod-file-table-ignore-sync.sql")}
 
-    {:name "0028-add-team-project-profile-rel-table"
-     :fn (mg/resource "app/migrations/sql/0028-add-team-project-profile-rel-table.sql")}
+   {:name "0028-add-team-project-profile-rel-table"
+    :fn (mg/resource "app/migrations/sql/0028-add-team-project-profile-rel-table.sql")}
 
-    {:name "0029-del-project-profile-rel-indexes"
-     :fn (mg/resource "app/migrations/sql/0029-del-project-profile-rel-indexes.sql")}
+   {:name "0029-del-project-profile-rel-indexes"
+    :fn (mg/resource "app/migrations/sql/0029-del-project-profile-rel-indexes.sql")}
 
-    {:name "0030-mod-file-table-add-missing-index"
-     :fn (mg/resource "app/migrations/sql/0030-mod-file-table-add-missing-index.sql")}
+   {:name "0030-mod-file-table-add-missing-index"
+    :fn (mg/resource "app/migrations/sql/0030-mod-file-table-add-missing-index.sql")}
 
-    {:name "0031-add-conversation-related-tables"
-     :fn (mg/resource "app/migrations/sql/0031-add-conversation-related-tables.sql")}
+   {:name "0031-add-conversation-related-tables"
+    :fn (mg/resource "app/migrations/sql/0031-add-conversation-related-tables.sql")}
 
-    {:name "0032-del-unused-tables"
-     :fn (mg/resource "app/migrations/sql/0032-del-unused-tables.sql")}
+   {:name "0032-del-unused-tables"
+    :fn (mg/resource "app/migrations/sql/0032-del-unused-tables.sql")}
 
-    {:name "0033-mod-comment-thread-table"
-     :fn (mg/resource "app/migrations/sql/0033-mod-comment-thread-table.sql")}
+   {:name "0033-mod-comment-thread-table"
+    :fn (mg/resource "app/migrations/sql/0033-mod-comment-thread-table.sql")}
 
-    {:name "0034-mod-profile-table-add-props-field"
-     :fn (mg/resource "app/migrations/sql/0034-mod-profile-table-add-props-field.sql")}
-    ]})
+   {:name "0034-mod-profile-table-add-props-field"
+    :fn (mg/resource "app/migrations/sql/0034-mod-profile-table-add-props-field.sql")}
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Entry point
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+   {:name "0035-add-storage-tables"
+    :fn (mg/resource "app/migrations/sql/0035-add-storage-tables.sql")}
 
-(defn migrate
-  []
-  (with-open [conn (db/open)]
-    (mg/setup! conn)
-    (mg/migrate! conn +migrations+)))
+   {:name "0036-mod-storage-referenced-tables"
+    :fn (mg/resource "app/migrations/sql/0036-mod-storage-referenced-tables.sql")}
 
-(defstate migrations
-  :start (migrate))
+   {:name "0037-del-obsolete-triggers"
+    :fn (mg/resource "app/migrations/sql/0037-del-obsolete-triggers.sql")}
+
+   {:name "0038-add-storage-on-delete-triggers"
+    :fn (mg/resource "app/migrations/sql/0038-add-storage-on-delete-triggers.sql")}
+
+   {:name "0039-fix-some-on-delete-triggers"
+    :fn (mg/resource "app/migrations/sql/0039-fix-some-on-delete-triggers.sql")}
+
+   {:name "0040-add-error-report-tables"
+    :fn (mg/resource "app/migrations/sql/0040-add-error-report-tables.sql")}
+
+   {:name "0041-mod-pg-storage-options"
+    :fn (mg/resource "app/migrations/sql/0041-mod-pg-storage-options.sql")}
+
+   {:name "0042-add-server-prop-table"
+    :fn (mg/resource "app/migrations/sql/0042-add-server-prop-table.sql")}
+
+   {:name "0043-drop-old-tables-and-fields"
+    :fn (mg/resource "app/migrations/sql/0043-drop-old-tables-and-fields.sql")}
+
+   {:name "0044-add-storage-refcount"
+    :fn (mg/resource "app/migrations/sql/0044-add-storage-refcount.sql")}
+
+   {:name "0045-add-index-to-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0045-add-index-to-file-change-table.sql")}
+
+   {:name "0046-add-profile-complaint-table"
+    :fn (mg/resource "app/migrations/sql/0046-add-profile-complaint-table.sql")}
+
+   {:name "0047-mod-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0047-mod-file-change-table.sql")}
+
+   {:name "0048-mod-storage-tables"
+    :fn (mg/resource "app/migrations/sql/0048-mod-storage-tables.sql")}
+
+   {:name "0049-mod-http-session-table"
+    :fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")}
+
+   {:name "0050-mod-server-prop-table"
+    :fn (mg/resource "app/migrations/sql/0050-mod-server-prop-table.sql")}
+
+   {:name "0051-mod-file-library-rel-table"
+    :fn (mg/resource "app/migrations/sql/0051-mod-file-library-rel-table.sql")}
+
+   {:name "0052-del-legacy-user-and-team"
+    :fn (mg/resource "app/migrations/sql/0052-del-legacy-user-and-team.sql")}
+
+   {:name "0053-add-team-font-variant-table"
+    :fn (mg/resource "app/migrations/sql/0053-add-team-font-variant-table.sql")}
+
+   {:name "0054-add-audit-log-table"
+    :fn (mg/resource "app/migrations/sql/0054-add-audit-log-table.sql")}
+
+   {:name "0055-mod-file-media-object-table"
+    :fn (mg/resource "app/migrations/sql/0055-mod-file-media-object-table.sql")}
+   ])
+
+
+(defmethod ig/init-key ::migrations [_ _] migrations)

backend/src/app/migrations/migration_0023.clj

@@ -2,10 +2,7 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; This Source Code Form is "Incompatible With Secondary Licenses", as
-;; defined by the Mozilla Public License, v. 2.0.
-;;
-;; Copyright (c) 2020 UXBOX Labs SL
+;; Copyright (c) UXBOX Labs SL
 
 (ns app.migrations.migration-0023
   (:require

backend/src/app/migrations/sql/0001-add-extensions.sql

@@ -1,5 +1,4 @@
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-CREATE EXTENSION IF NOT EXISTS "pgcrypto";
 
 CREATE FUNCTION update_modified_at()
   RETURNS TRIGGER AS $updt$

backend/src/app/migrations/sql/0003-add-project-tables.sql

@@ -106,8 +106,6 @@ CREATE TRIGGER file_image__on_delete__tgr
  AFTER DELETE ON file_image
    FOR EACH ROW EXECUTE PROCEDURE handle_delete();
 
-
-
 CREATE TABLE page (
   id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
   file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE,

backend/src/app/migrations/sql/0015-improve-tasks-tables.sql

@@ -1,4 +1,4 @@
-DROP TABLE task;
+DROP TABLE IF EXISTS task;
 
 CREATE TABLE task (
   id uuid DEFAULT uuid_generate_v4(),
@@ -27,3 +27,11 @@ CREATE TABLE task_default partition OF task default;
 CREATE INDEX task__scheduled_at__queue__idx
     ON task (scheduled_at, queue)
  WHERE status = 'new' or status = 'retry';
+
+ALTER TABLE task
+  ALTER COLUMN queue SET STORAGE external,
+  ALTER COLUMN name SET STORAGE external,
+  ALTER COLUMN props SET STORAGE external,
+  ALTER COLUMN status SET STORAGE external,
+  ALTER COLUMN error SET STORAGE external;
+

backend/src/app/migrations/sql/0019-add-improved-scheduled-tasks.sql

@@ -1,4 +1,4 @@
-DROP TABLE scheduled_task;
+DROP TABLE IF EXISTS scheduled_task;
 
 CREATE TABLE scheduled_task (
   id text PRIMARY KEY,
@@ -22,3 +22,7 @@ CREATE TABLE scheduled_task_history (
 
 CREATE INDEX scheduled_task_history__task_id__idx
     ON scheduled_task_history(task_id);
+
+ALTER TABLE scheduled_task
+  ALTER COLUMN id SET STORAGE external,
+  ALTER COLUMN cron_expr SET STORAGE external;

backend/src/app/migrations/sql/0035-add-storage-tables.sql

@@ -0,0 +1,36 @@
+CREATE TABLE storage_object (
+  id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  deleted_at timestamptz NULL DEFAULT NULL,
+
+  size bigint NOT NULL DEFAULT 0,
+  backend text NOT NULL,
+
+  metadata jsonb NULL DEFAULT NULL
+);
+
+CREATE INDEX storage_object__id__deleted_at__idx
+    ON storage_object(id, deleted_at)
+ WHERE deleted_at IS NOT null;
+
+CREATE TABLE storage_data (
+  id uuid PRIMARY KEY REFERENCES storage_object (id) ON DELETE CASCADE,
+  data bytea NOT NULL
+);
+
+CREATE INDEX storage_data__id__idx ON storage_data(id);
+
+-- Table used for store inflight upload ids, for later recheck and
+-- delete possible staled files that exists on the phisical storage
+-- but does not exists in the 'storage_object' table.
+
+CREATE TABLE storage_pending (
+  id uuid NOT NULL,
+
+  backend text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now(),
+
+  PRIMARY KEY (created_at, id)
+);
+

backend/src/app/migrations/sql/0036-mod-storage-referenced-tables.sql

@@ -0,0 +1,22 @@
+-- Profile
+ALTER TABLE profile ADD COLUMN photo_id uuid NULL REFERENCES storage_object(id) ON DELETE SET NULL;
+CREATE INDEX profile__photo_id__idx ON profile(photo_id);
+
+-- Team
+ALTER TABLE team ADD COLUMN photo_id uuid NULL REFERENCES storage_object(id) ON DELETE SET NULL;
+CREATE INDEX team__photo_id__idx ON team(photo_id);
+
+-- Media Objects -> File Media Objects
+ALTER TABLE media_object RENAME TO file_media_object;
+ALTER TABLE media_thumbnail RENAME TO file_media_thumbnail;
+
+ALTER TABLE file_media_object
+  ADD COLUMN media_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE,
+  ADD COLUMN thumbnail_id uuid NULL REFERENCES storage_object(id) ON DELETE CASCADE;
+
+CREATE INDEX file_media_object__image_id__idx ON file_media_object(media_id);
+CREATE INDEX file_media_object__thumbnail_id__idx ON file_media_object(thumbnail_id);
+
+ALTER TABLE file_media_object ALTER COLUMN path DROP NOT NULL;
+ALTER TABLE profile ALTER COLUMN photo DROP NOT NULL;
+ALTER TABLE team ALTER COLUMN photo DROP NOT NULL;

backend/src/app/migrations/sql/0037-del-obsolete-triggers.sql

@@ -0,0 +1,3 @@
+DROP FUNCTION update_modified_at () CASCADE;
+DROP FUNCTION handle_delete ( ) CASCADE;
+DROP TABLE pending_to_delete;

backend/src/app/migrations/sql/0038-add-storage-on-delete-triggers.sql

@@ -0,0 +1,50 @@
+CREATE FUNCTION on_delete_profile()
+  RETURNS TRIGGER AS $func$
+  BEGIN
+    UPDATE storage_object
+       SET deleted_at = now()
+     WHERE id = OLD.photo_id;
+
+    RETURN OLD;
+  END;
+$func$ LANGUAGE plpgsql;
+
+CREATE FUNCTION on_delete_team()
+  RETURNS TRIGGER AS $func$
+  BEGIN
+    UPDATE storage_object
+       SET deleted_at = now()
+     WHERE id = OLD.photo_id;
+
+    RETURN OLD;
+  END;
+$func$ LANGUAGE plpgsql;
+
+CREATE FUNCTION on_delete_file_media_object()
+  RETURNS TRIGGER AS $func$
+  BEGIN
+    UPDATE storage_object
+       SET deleted_at = now()
+     WHERE id = OLD.media_id;
+
+    IF OLD.thumbnail_id IS NOT NULL THEN
+      UPDATE storage_object
+         SET deleted_at = now()
+       WHERE id = OLD.thumbnail_id;
+    END IF;
+
+    RETURN OLD;
+  END;
+$func$ LANGUAGE plpgsql;
+
+CREATE TRIGGER profile__on_delete__tgr
+ AFTER DELETE ON profile
+   FOR EACH ROW EXECUTE PROCEDURE on_delete_profile();
+
+CREATE TRIGGER team__on_delete__tgr
+ AFTER DELETE ON team
+   FOR EACH ROW EXECUTE PROCEDURE on_delete_team();
+
+CREATE TRIGGER file_media_object__on_delete__tgr
+ AFTER DELETE ON file_media_object
+   FOR EACH ROW EXECUTE PROCEDURE on_delete_file_media_object();

backend/src/app/migrations/sql/0039-fix-some-on-delete-triggers.sql

@@ -0,0 +1,9 @@
+ALTER TABLE file_library_rel
+ DROP CONSTRAINT file_library_rel_library_file_id_fkey,
+  ADD CONSTRAINT file_library_rel_library_file_id_fkey
+      FOREIGN KEY (library_file_id) REFERENCES file(id) ON DELETE CASCADE;
+
+ALTER TABLE team_profile_rel
+ DROP CONSTRAINT team_profile_rel_profile_id_fkey,
+  ADD CONSTRAINT team_profile_rel_profile_id_fkey
+      FOREIGN KEY (profile_id) REFERENCES profile(id) ON DELETE CASCADE;

backend/src/app/migrations/sql/0040-add-error-report-tables.sql

@@ -0,0 +1,10 @@
+CREATE TABLE server_error_report (
+  id uuid NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+  content jsonb,
+
+  PRIMARY KEY (id, created_at)
+);
+
+ALTER TABLE server_error_report
+  ALTER COLUMN content SET STORAGE external;

backend/src/app/migrations/sql/0041-mod-pg-storage-options.sql

@@ -0,0 +1,46 @@
+ALTER TABLE file
+  ALTER COLUMN data SET STORAGE external,
+  ALTER COLUMN name SET STORAGE external;
+
+ALTER TABLE file_change
+  ALTER COLUMN data SET STORAGE external,
+  ALTER COLUMN changes SET STORAGE external;
+
+ALTER TABLE profile
+  ALTER COLUMN fullname SET STORAGE external,
+  ALTER COLUMN email SET STORAGE external,
+  ALTER COLUMN password SET STORAGE external,
+  ALTER COLUMN lang SET STORAGE external,
+  ALTER COLUMN theme SET STORAGE external,
+  ALTER COLUMN props SET STORAGE external;
+
+ALTER TABLE project
+  ALTER COLUMN name SET STORAGE external;
+
+ALTER TABLE team
+  ALTER COLUMN name SET STORAGE external;
+
+ALTER TABLE comment
+  ALTER COLUMN content SET STORAGE external;
+
+ALTER TABLE comment_thread
+  ALTER COLUMN participants SET STORAGE external,
+  ALTER COLUMN page_name SET STORAGE external;
+
+ALTER TABLE http_session
+  ALTER COLUMN id SET STORAGE external,
+  ALTER COLUMN user_agent SET STORAGE external;
+
+ALTER TABLE file_share_token
+  ALTER COLUMN token SET STORAGE external;
+
+ALTER TABLE file_media_object
+  ALTER COLUMN name SET STORAGE external,
+  ALTER COLUMN mtype SET STORAGE external;
+
+ALTER TABLE storage_object
+  ALTER COLUMN backend SET STORAGE external,
+  ALTER COLUMN metadata SET STORAGE external;
+
+ALTER TABLE storage_data
+  ALTER COLUMN data SET STORAGE external;

backend/src/app/migrations/sql/0042-add-server-prop-table.sql

@@ -0,0 +1,8 @@
+CREATE TABLE server_prop (
+  id text PRIMARY KEY,
+  content jsonb
+);
+
+ALTER TABLE server_prop
+  ALTER COLUMN id SET STORAGE external,
+  ALTER COLUMN content SET STORAGE external;

backend/src/app/migrations/sql/0043-drop-old-tables-and-fields.sql

@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS file_media_thumbnail;
+
+ALTER TABLE profile DROP COLUMN photo;
+ALTER TABLE team DROP COLUMN photo;
+
+ALTER TABLE file_media_object DROP COLUMN path;
+ALTER TABLE file_media_object ALTER COLUMN media_id SET NOT NULL;
+
+ALTER TRIGGER media_object__insert__tgr
+   ON file_media_object RENAME TO file_media_object__on_insert__tgr;

backend/src/app/migrations/sql/0044-add-storage-refcount.sql

@@ -0,0 +1,23 @@
+ALTER TABLE storage_object
+  ADD COLUMN touched_at timestamptz NULL;
+
+CREATE INDEX storage_object__id_touched_at__idx
+    ON storage_object (touched_at, id)
+ WHERE touched_at IS NOT NULL;
+
+CREATE OR REPLACE FUNCTION on_delete_file_media_object()
+  RETURNS TRIGGER AS $func$
+  BEGIN
+    IF OLD.thumbnail_id IS NOT NULL THEN
+      UPDATE storage_object
+         SET touched_at = now()
+       WHERE id in (OLD.thumbnail_id, OLD.media_id);
+    ELSE
+      UPDATE storage_object
+         SET touched_at = now()
+       WHERE id = OLD.media_id;
+    END IF;
+    RETURN OLD;
+  END;
+$func$ LANGUAGE plpgsql;
+

backend/src/app/migrations/sql/0045-add-index-to-file-change-table.sql

@@ -0,0 +1,2 @@
+CREATE INDEX file_change__created_at_idx
+    ON file_change (created_at);