📎 Update changelog

✨ Ip Addr parsing and audit log context forwarding fixes

CHANGES.md

@@ -1,6 +1,19 @@
 # CHANGELOG
 
-## 2.1.0
+## 2.1.1
+
+### :sparkles: New features
+
+- Consolidate templates new order and naming  [Taiga #8392](https://tree.taiga.io/project/penpot/task/8392)
+
+### :bug: Bugs fixed
+
+- Fix the “search” label in translations [Taiga #8402](https://tree.taiga.io/project/penpot/issue/8402)
+- Fix pencil loader [Taiga #8348](https://tree.taiga.io/project/penpot/issue/8348)
+- Fix several issues on the OIDC.
+- Fix regression on the `email-verification` flag [Taiga #8398](https://tree.taiga.io/project/penpot/issue/8398)
+
+## 2.1.0 - Things can only get better!
 
 ### :rocket: Epics and highlights
 
@@ -54,6 +67,7 @@
 - Fix 'Detach instance' shortcut is not working [Taiga #8102](https://tree.taiga.io/project/penpot/issue/8102)
 - Fix import file message does not detect 0 as error [Taiga #6824](https://tree.taiga.io/project/penpot/issue/6824)
 - Image Color Library is not persisted when exporting/importing in .zip [Taiga #8131](https://tree.taiga.io/project/penpot/issue/8131)
+- Fix export files including libraries [Taiga #8266](https://tree.taiga.io/project/penpot/issue/8266)
 
 ## 2.0.3
 

backend/resources/app/onboarding.edn

@@ -1,4 +1,16 @@
-[{:id "tutorial-for-beginners"
+[{:id "wireframing-kit"
+  :name "Wireframe library"
+  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/wireframing-kit.penpot"}
+ {:id "prototype-examples"
+  :name "Prototipe template"
+  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/prototype-examples.penpot"}
+ {:id "plants-app"
+  :name "UI mockup example"
+  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Plants-app.penpot"}
+ {:id "penpot-design-system"
+  :name "Design system example"
+  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Penpot-Design-system.penpot"}
+ {:id "tutorial-for-beginners"
   :name "Tutorial for beginners"
   :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/tutorial-for-beginners.penpot"}
  {:id "lucide-icons"
@@ -7,12 +19,6 @@
  {:id "font-awesome"
   :name "Font Awesome"
   :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Font-Awesome.penpot"}
- {:id "plants-app"
-  :name "Plants app"
-  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Plants-app.penpot"}
- {:id "wireframing-kit"
-  :name "Wireframing Kit"
-  :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/wireframing-kit.penpot"}
  {:id "black-white-mobile-templates"
   :name "Black & White Mobile Templates"
   :file-uri "https://github.com/penpot/penpot-files/raw/binary-files/Black-White-Mobile-Templates.penpot"}

backend/src/app/auth/oidc.clj

@@ -19,11 +19,14 @@
    [app.email.blacklist :as email.blacklist]
    [app.email.whitelist :as email.whitelist]
    [app.http.client :as http]
+   [app.http.errors :as errors]
    [app.http.session :as session]
    [app.loggers.audit :as audit]
+   [app.rpc :as rpc]
    [app.rpc.commands.profile :as profile]
    [app.setup :as-alias setup]
    [app.tokens :as tokens]
+   [app.util.inet :as inet]
    [app.util.json :as json]
    [app.util.time :as dt]
    [buddy.sign.jwk :as jwk]
@@ -130,8 +133,8 @@
           (-> body json/decode :keys process-oidc-jwks)
           (do
             (l/warn :hint "unable to retrieve JWKs (unexpected response status code)"
-                    :http-status status
-                    :http-body  body)
+                    :response-status status
+                    :response-body  body)
             nil)))
       (catch Throwable cause
         (l/warn :hint "unable to retrieve JWKs (unexpected exception)"
@@ -145,18 +148,18 @@
   (when (contains? cf/flags :login-with-oidc)
     (if-let [opts (prepare-oidc-opts cfg)]
       (let [jwks (fetch-oidc-jwks cfg opts)]
-        (l/info :hint "provider initialized"
-                :provider "oidc"
-                :method (if (:discover? opts) "discover" "manual")
-                :client-id (:client-id opts)
-                :client-secret (obfuscate-string (:client-secret opts))
-                :scopes     (str/join "," (:scopes opts))
-                :auth-uri   (:auth-uri opts)
-                :user-uri   (:user-uri opts)
-                :token-uri  (:token-uri opts)
-                :roles-attr (:roles-attr opts)
-                :roles      (:roles opts)
-                :keys       (str/join "," (map str (keys jwks))))
+        (l/inf :hint "provider initialized"
+               :provider "oidc"
+               :method (if (:discover? opts) "discover" "manual")
+               :client-id (:client-id opts)
+               :client-secret (obfuscate-string (:client-secret opts))
+               :scopes     (str/join "," (:scopes opts))
+               :auth-uri   (:auth-uri opts)
+               :user-uri   (:user-uri opts)
+               :token-uri  (:token-uri opts)
+               :roles-attr (:roles-attr opts)
+               :roles      (:roles opts)
+               :keys       (str/join "," (map str (keys jwks))))
         (assoc opts :jwks jwks))
       (do
         (l/warn :hint "unable to initialize auth provider, missing configuration" :provider "oidc")
@@ -180,10 +183,10 @@
       (if (and (string? (:client-id opts))
                (string? (:client-secret opts)))
         (do
-          (l/info :hint "provider initialized"
-                  :provider "google"
-                  :client-id (:client-id opts)
-                  :client-secret (obfuscate-string (:client-secret opts)))
+          (l/inf :hint "provider initialized"
+                 :provider "google"
+                 :client-id (:client-id opts)
+                 :client-secret (obfuscate-string (:client-secret opts)))
           opts)
 
         (do
@@ -208,8 +211,9 @@
           (ex/raise :type :internal
                     :code :unable-to-retrieve-github-emails
                     :hint "unable to retrieve github emails"
-                    :http-status status
-                    :http-body body))
+                    :request-uri (:uri params)
+                    :response-status status
+                    :response-body body))
 
         (->> body json/decode (filter :primary) first :email))))
 
@@ -234,10 +238,10 @@
       (if (and (string? (:client-id opts))
                (string? (:client-secret opts)))
         (do
-          (l/info :hint "provider initialized"
-                  :provider "github"
-                  :client-id (:client-id opts)
-                  :client-secret (obfuscate-string (:client-secret opts)))
+          (l/inf :hint "provider initialized"
+                 :provider "github"
+                 :client-id (:client-id opts)
+                 :client-secret (obfuscate-string (:client-secret opts)))
           opts)
 
         (do
@@ -249,7 +253,7 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defmethod ig/init-key ::providers/gitlab
-  [_ _]
+  [_ cfg]
   (let [base (cf/get :gitlab-base-uri "https://gitlab.com")
         opts {:base-uri      base
               :client-id     (cf/get :gitlab-client-id)
@@ -258,17 +262,18 @@
               :auth-uri      (str base "/oauth/authorize")
               :token-uri     (str base "/oauth/token")
               :user-uri      (str base "/oauth/userinfo")
+              :jwks-uri      (str base "/oauth/discovery/keys")
               :name          "gitlab"}]
     (when (contains? cf/flags :login-with-gitlab)
       (if (and (string? (:client-id opts))
                (string? (:client-secret opts)))
-        (do
-          (l/info :hint "provider initialized"
-                  :provider "gitlab"
-                  :base-uri base
-                  :client-id (:client-id opts)
-                  :client-secret (obfuscate-string (:client-secret opts)))
-          opts)
+        (let [jwks (fetch-oidc-jwks cfg opts)]
+          (l/inf :hint "provider initialized"
+                 :provider "gitlab"
+                 :base-uri base
+                 :client-id (:client-id opts)
+                 :client-secret (obfuscate-string (:client-secret opts)))
+          (assoc opts :jwks jwks))
 
         (do
           (l/warn :hint "unable to initialize auth provider, missing configuration" :provider "gitlab")
@@ -324,26 +329,31 @@
                 :uri (:token-uri provider)
                 :body (u/map->query-string params)}]
 
-    (l/trace :hint "request access token"
-             :provider (:name provider)
-             :client-id (:client-id provider)
-             :client-secret (obfuscate-string (:client-secret provider))
-             :grant-type (:grant_type params)
-             :redirect-uri (:redirect_uri params))
+    (l/trc :hint "fetch access token"
+           :provider (:name provider)
+           :client-id (:client-id provider)
+           :client-secret (obfuscate-string (:client-secret provider))
+           :grant-type (:grant_type params)
+           :redirect-uri (:redirect_uri params))
 
     (let [{:keys [status body]} (http/req! cfg req {:sync? true})]
-      (l/trace :hint "access token response" :status status :body body)
+      (l/trc :hint "access token fetched" :status status :body body)
       (if (= status 200)
-        (let [data (json/decode body)]
-          {:token/access (get data :access_token)
-           :token/id     (get data :id_token)
-           :token/type   (get data :token_type)})
-
+        (let [data (json/decode body)
+              data {:token/access (get data :access_token)
+                    :token/id     (get data :id_token)
+                    :token/type   (get data :token_type)}]
+          (l/trc :hint "access token fetched"
+                 :token-id (:token/id data)
+                 :token-type (:token/type data)
+                 :token (:token/access data))
+          data)
         (ex/raise :type :internal
-                  :code :unable-to-retrieve-token
-                  :hint "unable to retrieve token"
-                  :http-status status
-                  :http-body body)))))
+                  :code :unable-to-fetch-access-token
+                  :hint "unable to fetch access token"
+                  :request-uri (:uri req)
+                  :response-status status
+                  :response-body body)))))
 
 (defn- process-user-info
   [provider tdata info]
@@ -370,9 +380,9 @@
 
 (defn- fetch-user-info
   [{:keys [::provider] :as cfg} tdata]
-  (l/trace :hint "fetch user info"
-           :uri (:user-uri provider)
-           :token (obfuscate-string (:token/access tdata)))
+  (l/trc :hint "fetch user info"
+         :uri (:user-uri provider)
+         :token (obfuscate-string (:token/access tdata)))
 
   (let [params   {:uri (:user-uri provider)
                   :headers {"Authorization" (str (:token/type tdata) " " (:token/access tdata))}
@@ -380,9 +390,9 @@
                   :method :get}
         response (http/req! cfg params {:sync? true})]
 
-    (l/trace :hint "user info response"
-             :status (:status response)
-             :body   (:body response))
+    (l/trc :hint "user info response"
+           :status (:status response)
+           :body   (:body response))
 
     (when-not (s/int-in-range? 200 300 (:status response))
       (ex/raise :type :internal
@@ -420,12 +430,6 @@
 
 (defn- get-info
   [{:keys [::provider ::setup/props] :as cfg} {:keys [params] :as request}]
-  (when-let [error (get params :error)]
-    (ex/raise :type :internal
-              :code :error-on-retrieving-code
-              :error-id error
-              :error-desc (get params :error_description)))
-
   (let [state  (get params :state)
         code   (get params :code)
         state  (tokens/verify props {:token state :iss :oauth})
@@ -438,7 +442,7 @@
 
         info   (process-user-info provider tdata info)]
 
-    (l/trace :hint "user info" :info info)
+    (l/trc :hint "user info" :info info)
 
     (when-not (s/valid? ::info info)
       (l/warn :hint "received incomplete profile info object (please set correct scopes)" :info info)
@@ -568,10 +572,10 @@
             props   (audit/profile->props profile)
             context (d/without-nils {:external-session-id (:external-session-id info)})]
 
-        (audit/submit! cfg {::audit/type "command"
+        (audit/submit! cfg {::audit/type "action"
                             ::audit/name "login-with-oidc"
                             ::audit/profile-id (:id profile)
-                            ::audit/ip-addr (audit/parse-client-ip request)
+                            ::audit/ip-addr (inet/parse-request request)
                             ::audit/props props
                             ::audit/context context})
 
@@ -592,28 +596,50 @@
         (redirect-to-register cfg info request)
         (redirect-with-error "registration-disabled")))))
 
+(defn- get-external-session-id
+  [request]
+  (let [session-id (rreq/get-header request "x-external-session-id")]
+    (when (string? session-id)
+      (if (or (> (count session-id) 256)
+              (= session-id "null")
+              (str/blank? session-id))
+        nil
+        session-id))))
+
 (defn- auth-handler
   [cfg {:keys [params] :as request}]
-  (let [props (audit/extract-utm-params params)
-        esid  (rreq/get-header request "x-external-session-id")
-        state (tokens/generate (::setup/props cfg)
-                               {:iss :oauth
-                                :invitation-token (:invitation-token params)
-                                :external-session-id esid
-                                :props props
-                                :exp (dt/in-future "4h")})
-        uri   (build-auth-uri cfg state)]
+  (let [props  (audit/extract-utm-params params)
+        esid   (rpc/get-external-session-id request)
+        params {:iss :oauth
+                :invitation-token (:invitation-token params)
+                :external-session-id esid
+                :props props
+                :exp (dt/in-future "4h")}
+        state  (tokens/generate (::setup/props cfg)
+                                (d/without-nils params))
+        uri    (build-auth-uri cfg state)]
     {::rres/status 200
      ::rres/body {:redirect-uri uri}}))
 
 (defn- callback-handler
-  [cfg request]
+  [{:keys [::provider] :as cfg} request]
   (try
-    (let [info    (get-info cfg request)
-          profile (get-profile cfg info)]
-      (process-callback cfg request info profile))
+    (if-let [error (dm/get-in request [:params :error])]
+      (redirect-with-error "unable-to-auth" error)
+      (let [info    (get-info cfg request)
+            profile (get-profile cfg info)]
+        (process-callback cfg request info profile)))
     (catch Throwable cause
-      (l/err :hint "error on oauth process" :cause cause)
+      (binding [l/*context* (-> (errors/request->context request)
+                                (assoc :auth/provider (:name provider)))]
+        (let [edata (ex-data cause)]
+          (cond
+            (= :validation (:type edata))
+            (l/wrn :hint "invalid token received" :cause cause)
+
+            :else
+            (l/err :hint "error on oauth process" :cause cause))))
+
       (redirect-with-error "unable-to-auth" (ex-message cause)))))
 
 (def provider-lookup

backend/src/app/binfile/common.clj

@@ -15,6 +15,7 @@
    [app.common.files.migrations :as fmg]
    [app.common.files.validate :as fval]
    [app.common.logging :as l]
+   [app.common.types.file :as ctf]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
@@ -331,54 +332,12 @@
 
 (defn embed-assets
   [cfg data file-id]
-  (letfn [(walk-map-form [form state]
-            (cond
-              (uuid? (:fill-color-ref-file form))
-              (do
-                (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)])
-                (assoc form :fill-color-ref-file file-id))
-
-              (uuid? (:stroke-color-ref-file form))
-              (do
-                (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)])
-                (assoc form :stroke-color-ref-file file-id))
-
-              (uuid? (:typography-ref-file form))
-              (do
-                (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)])
-                (assoc form :typography-ref-file file-id))
-
-              (uuid? (:component-file form))
-              (do
-                (vswap! state conj [(:component-file form) :components (:component-id form)])
-                (assoc form :component-file file-id))
-
-              :else
-              form))
-
-          (process-group-of-assets [data [lib-id items]]
-            ;; NOTE: there is a possibility that shape refers to an
-            ;; non-existant file because the file was removed. In this
-            ;; case we just ignore the asset.
-            (if-let [lib (get-file cfg lib-id)]
-              (reduce (partial process-asset lib) data items)
-              data))
-
-          (process-asset [lib data [bucket asset-id]]
-            (let [asset (get-in lib [:data bucket asset-id])
-                  ;; Add a special case for colors that need to have
-                  ;; correctly set the :file-id prop (pending of the
-                  ;; refactor that will remove it).
-                  asset (cond-> asset
-                          (= bucket :colors) (assoc :file-id file-id))]
-              (update data bucket assoc asset-id asset)))]
-
-    (let [assets (volatile! [])]
-      (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data)
-      (->> (deref assets)
-           (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id))))
-           (d/group-by first rest)
-           (reduce (partial process-group-of-assets) data)))))
+  (let [library-ids (get-libraries cfg [file-id])]
+    (reduce (fn [data library-id]
+              (let [library (get-file cfg library-id)]
+                (ctf/absorb-assets data (:data library))))
+            data
+            library-ids)))
 
 (defn- fix-version
   [file]

backend/src/app/binfile/v1.clj

@@ -130,7 +130,6 @@
   (.writeLong output (long data))
   (swap! *position* + 8))
 
-
 (defn read-long!
   [^DataInputStream input]
   (let [v (.readLong input)]

backend/src/app/loggers/audit.clj

@@ -21,24 +21,18 @@
    [app.rpc :as-alias rpc]
    [app.rpc.retry :as rtry]
    [app.setup :as-alias setup]
+   [app.util.inet :as inet]
    [app.util.services :as-alias sv]
    [app.util.time :as dt]
    [app.worker :as wrk]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
-   [integrant.core :as ig]
-   [ring.request :as rreq]))
+   [integrant.core :as ig]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; HELPERS
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn parse-client-ip
-  [request]
-  (or (some-> (rreq/get-header request "x-forwarded-for") (str/split ",") first)
-      (rreq/get-header request "x-real-ip")
-      (some-> (rreq/remote-addr request) str)))
-
 (defn extract-utm-params
   "Extracts additional data from params and namespace them under
   `penpot` ns."
@@ -86,17 +80,20 @@
          (remove #(contains? reserved-props (key %))))
         props))
 
-(defn params->context
-  "Extract default context properties from RPC params object"
+(defn event-from-rpc-params
+  "Create a base event skeleton with pre-filled some important
+  data that can be extracted from RPC params object"
   [params]
-  (d/without-nils
-   {:external-session-id (::rpc/external-session-id params)
-    :event-origin (::rpc/external-event-origin params)
-    :triggered-by (::rpc/handler-name params)}))
+  (let [context {:external-session-id (::rpc/external-session-id params)
+                 :external-event-origin (::rpc/external-event-origin params)
+                 :triggered-by (::rpc/handler-name params)}]
+    {::type "action"
+     ::profile-id (::rpc/profile-id params)
+     ::ip-addr (::rpc/ip-addr params)
+     ::context (d/without-nils context)}))
 
 ;; --- SPECS
 
-
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; COLLECTOR
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -163,14 +160,16 @@
                       (assoc :external-session-id session-id)
                       (assoc :external-event-origin event-origin)
                       (assoc :access-token-id (some-> token-id str))
-                      (d/without-nils))]
+                      (d/without-nils))
+
+        ip-addr   (inet/parse-request request)]
 
     {::type (or (::type resultm)
                 (::rpc/type cfg))
      ::name (or (::name resultm)
                 (::sv/name mdata))
      ::profile-id profile-id
-     ::ip-addr (some-> request parse-client-ip)
+     ::ip-addr ip-addr
      ::props props
      ::context context
 
@@ -192,15 +191,33 @@
          (::webhooks/event? resultm)
          false)}))
 
-(defn- handle-event!
-  [cfg event]
+(defn- event->params
+  [event]
   (let [params {:id (uuid/next)
                 :name (::name event)
                 :type (::type event)
                 :profile-id (::profile-id event)
                 :ip-addr (::ip-addr event)
-                :context (::context event)
-                :props (::props event)}
+                :context (::context event {})
+                :props (::props event {})
+                :source "backend"}
+        tnow   (::tracked-at event)]
+
+    (cond-> params
+      (some? tnow)
+      (assoc :tracked-at tnow))))
+
+(defn- append-audit-entry!
+  [cfg params]
+  (let [params (-> params
+                   (update :props db/tjson)
+                   (update :context db/tjson)
+                   (update :ip-addr db/inet))]
+    (db/insert! cfg :audit-log params)))
+
+(defn- handle-event!
+  [cfg event]
+  (let [params (event->params event)
         tnow   (dt/now)]
 
     (when (contains? cf/flags :audit-log)
@@ -209,12 +226,8 @@
       ;; this case we just retry the operation.
       (let [params (-> params
                        (assoc :created-at tnow)
-                       (assoc :tracked-at tnow)
-                       (update :props db/tjson)
-                       (update :context db/tjson)
-                       (update :ip-addr db/inet)
-                       (assoc :source "backend"))]
-        (db/insert! cfg :audit-log params)))
+                       (update :tracked-at #(or % tnow)))]
+        (append-audit-entry! cfg params)))
 
     (when (and (or (contains? cf/flags :telemetry)
                    (cf/get :telemetry-enabled))
@@ -226,12 +239,10 @@
       ;; NOTE: this is only executed when general audit log is disabled
       (let [params (-> params
                        (assoc :created-at tnow)
-                       (assoc :tracked-at tnow)
-                       (assoc :props (db/tjson {}))
-                       (assoc :context (db/tjson {}))
-                       (assoc :ip-addr (db/inet "0.0.0.0"))
-                       (assoc :source "backend"))]
-        (db/insert! cfg :audit-log params)))
+                       (update :tracked-at #(or % tnow))
+                       (assoc :props {})
+                       (assoc :context {}))]
+        (append-audit-entry! cfg params)))
 
     (when (and (contains? cf/flags :webhooks)
                (::webhooks/event? event))
@@ -258,9 +269,9 @@
 
 (defn submit!
   "Submit audit event to the collector."
-  [cfg params]
+  [cfg event]
   (try
-    (let [event (d/without-nils params)
+    (let [event (d/without-nils event)
           cfg   (-> cfg
                     (assoc ::rtry/when rtry/conflict-exception?)
                     (assoc ::rtry/max-retries 6)
@@ -269,3 +280,18 @@
       (rtry/invoke! cfg db/tx-run! handle-event! event))
     (catch Throwable cause
       (l/error :hint "unexpected error processing event" :cause cause))))
+
+(defn insert!
+  "Submit audit event to the collector, intended to be used only from
+  command line helpers because this skips all webhooks and telemetry
+  logic."
+  [cfg event]
+  (when (contains? cf/flags :audit-log)
+    (let [event (d/without-nils event)]
+      (us/verify! ::event event)
+      (db/run! cfg (fn [cfg]
+                     (let [tnow   (dt/now)
+                           params (-> (event->params event)
+                                      (assoc :created-at tnow)
+                                      (update :tracked-at #(or % tnow)))]
+                       (append-audit-entry! cfg params)))))))

backend/src/app/main.clj

@@ -254,7 +254,7 @@
    {::http.client/client (ig/ref ::http.client/client)}
 
    ::oidc.providers/gitlab
-   {}
+   {::http.client/client (ig/ref ::http.client/client)}
 
    ::oidc.providers/generic
    {::http.client/client (ig/ref ::http.client/client)}

backend/src/app/rpc.clj

@@ -29,6 +29,7 @@
    [app.rpc.rlimit :as rlimit]
    [app.setup :as-alias setup]
    [app.storage :as-alias sto]
+   [app.util.inet :as inet]
    [app.util.services :as sv]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
@@ -70,6 +71,22 @@
         (handle-response-transformation request mdata)
         (handle-before-comple-hook mdata))))
 
+(defn get-external-session-id
+  [request]
+  (when-let [session-id (rreq/get-header request "x-external-session-id")]
+    (when-not (or (> (count session-id) 256)
+                  (= session-id "null")
+                  (str/blank? session-id))
+      session-id)))
+
+(defn- get-external-event-origin
+  [request]
+  (when-let [origin (rreq/get-header request "x-event-origin")]
+    (when-not (or (> (count origin) 256)
+                  (= origin "null")
+                  (str/blank? origin))
+      origin)))
+
 (defn- rpc-handler
   "Ring handler that dispatches cmd requests and convert between
   internal async flow into ring async flow."
@@ -79,11 +96,13 @@
         profile-id   (or (::session/profile-id request)
                          (::actoken/profile-id request))
 
-        session-id   (rreq/get-header request "x-external-session-id")
-        event-origin (rreq/get-header request "x-event-origin")
+        ip-addr      (inet/parse-request request)
+        session-id   (get-external-session-id request)
+        event-origin (get-external-event-origin request)
 
         data         (-> params
                          (assoc ::handler-name handler-name)
+                         (assoc ::ip-addr ip-addr)
                          (assoc ::request-at (dt/now))
                          (assoc ::external-session-id session-id)
                          (assoc ::external-event-origin event-origin)

backend/src/app/rpc/commands/audit.clj

@@ -14,11 +14,12 @@
    [app.config :as cf]
    [app.db :as db]
    [app.http :as-alias http]
-   [app.loggers.audit :as audit]
+   [app.loggers.audit :as-alias audit]
    [app.rpc :as-alias rpc]
    [app.rpc.climit :as-alias climit]
    [app.rpc.doc :as-alias doc]
    [app.rpc.helpers :as rph]
+   [app.util.inet :as inet]
    [app.util.services :as sv]
    [app.util.time :as dt]))
 
@@ -61,7 +62,7 @@
 (defn- handle-events
   [{:keys [::db/pool]} {:keys [::rpc/profile-id events] :as params}]
   (let [request (-> params meta ::http/request)
-        ip-addr (audit/parse-client-ip request)
+        ip-addr (inet/parse-request request)
         tnow    (dt/now)
         xform   (comp
                  (map (fn [event]

backend/src/app/rpc/commands/auth.clj

@@ -340,7 +340,7 @@
 
         profile    (if-let [profile-id (:profile-id claims)]
                      (profile/get-profile conn profile-id)
-                     (let [is-active (or (boolean (:is-active params))
+                     (let [is-active (or (boolean (:is-active claims))
                                          (not (contains? cf/flags :email-verification)))
                            params    (-> params
                                          (assoc :is-active is-active)
@@ -348,6 +348,9 @@
                        (->> (create-profile! conn params)
                             (create-profile-rels! conn))))
 
+        ;; When no profile-id comes on claims means a new register
+        created?   (not (:profile-id claims))
+
         invitation (when-let [token (:invitation-token params)]
                      (tokens/verify (::setup/props cfg) {:token token :iss :team-invitation}))
 
@@ -385,8 +388,8 @@
       ;; When a new user is created and it is already activated by
       ;; configuration or specified by OIDC, we just mark the profile
       ;; as logged-in
-      (not (:profile-id claims))
-      (if (:is-active claims)
+      created?
+      (if (:is-active profile)
         (-> (profile/strip-private-attrs profile)
             (rph/with-transform (session/create-fn cfg (:id profile)))
             (rph/with-meta

backend/src/app/rpc/commands/management.clj

@@ -413,15 +413,13 @@
                                   {:modified-at (dt/now)}
                                   {:id project-id})
 
-                      (let [props   (audit/clean-props params)
-                            context (audit/params->context params)]
+                      (let [props (audit/clean-props params)]
                         (doseq [file-id result]
-                          (audit/submit! cfg
-                                         {::audit/type "action"
-                                          ::audit/name "create-file"
-                                          ::audit/profile-id profile-id
-                                          ::audit/props (assoc props :id file-id)
-                                          ::audit/context context})))
+                          (let [props (assoc props :id file-id)
+                                event (-> (audit/event-from-rpc-params params)
+                                          (assoc ::audit/name "create-file")
+                                          (assoc ::audit/props props))]
+                            (audit/submit! cfg event))))
 
                       result))))
 

backend/src/app/rpc/commands/teams.clj

@@ -787,18 +787,15 @@
           (l/info :hint "invitation token" :token itoken))
 
 
-        (let [props   (-> (dissoc tprops :profile-id)
-                          (audit/clean-props))
-              context (audit/params->context params)]
-
-          (audit/submit! cfg
-                         {::audit/type "action"
-                          ::audit/name (if updated?
-                                         "update-team-invitation"
-                                         "create-team-invitation")
-                          ::audit/profile-id (:id profile)
-                          ::audit/props props
-                          ::audit/context context}))
+        (let [props  (-> (dissoc tprops :profile-id)
+                         (audit/clean-props))
+              evname (if updated?
+                       "update-team-invitation"
+                       "create-team-invitation")
+              event (-> (audit/event-from-rpc-params params)
+                        (assoc ::audit/name evname)
+                        (assoc ::audit/props props))]
+          (audit/submit! cfg event))
 
         (eml/send! {::eml/conn conn
                     ::eml/factory eml/invite-to-team
@@ -882,62 +879,51 @@
 (sv/defmethod ::create-team-with-invitations
   {::doc/added "1.17"
    ::sm/params schema:create-team-with-invitations}
-  [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id emails role name] :as params}]
-  (db/with-atomic [conn pool]
+  [cfg {:keys [::rpc/profile-id emails role name] :as params}]
 
-    (let [features (-> (cfeat/get-enabled-features cf/flags)
-                       (cfeat/check-client-features! (:features params)))
+  (db/tx-run! cfg
+              (fn [{:keys [::db/conn] :as cfg}]
+                (let [features (-> (cfeat/get-enabled-features cf/flags)
+                                   (cfeat/check-client-features! (:features params)))
 
-          params   (-> params
-                       (assoc :profile-id profile-id)
-                       (assoc :features features))
+                      params   (-> params
+                                   (assoc :profile-id profile-id)
+                                   (assoc :features features))
 
-          cfg      (assoc cfg ::db/conn conn)
-          team     (create-team cfg params)
-          profile  (db/get-by-id conn :profile profile-id)
-          emails   (into #{} (map profile/clean-email) emails)
-          context  (audit/params->context params)]
+                      cfg      (assoc cfg ::db/conn conn)
+                      team     (create-team cfg params)
+                      profile  (db/get-by-id conn :profile profile-id)
+                      emails   (into #{} (map profile/clean-email) emails)]
 
-      ;; Create invitations for all provided emails.
-      (->> emails
-           (map (fn [email]
-                  (-> params
-                      (assoc :team team)
-                      (assoc :profile profile)
-                      (assoc :email email)
-                      (assoc :role role))))
-           (run! (partial create-invitation cfg)))
+                  (let [props {:name name :features features}
+                        event (-> (audit/event-from-rpc-params params)
+                                  (assoc ::audit/name "create-team")
+                                  (assoc ::audit/props props))]
+                    (audit/submit! cfg event))
 
-      (run! (partial quotes/check-quote! conn)
-            (list {::quotes/id ::quotes/teams-per-profile
-                   ::quotes/profile-id profile-id}
-                  {::quotes/id ::quotes/invitations-per-team
-                   ::quotes/profile-id profile-id
-                   ::quotes/team-id (:id team)
-                   ::quotes/incr (count emails)}
-                  {::quotes/id ::quotes/profiles-per-team
-                   ::quotes/profile-id profile-id
-                   ::quotes/team-id (:id team)
-                   ::quotes/incr (count emails)}))
+                  ;; Create invitations for all provided emails.
+                  (->> emails
+                       (map (fn [email]
+                              (-> params
+                                  (assoc :team team)
+                                  (assoc :profile profile)
+                                  (assoc :email email)
+                                  (assoc :role role))))
+                       (run! (partial create-invitation cfg)))
 
-      (audit/submit! cfg
-                     {::audit/type "action"
-                      ::audit/name "create-team"
-                      ::audit/profile-id profile-id
-                      ::audit/props {:name name
-                                     :features features}
-                      ::audit/context context})
+                  (run! (partial quotes/check-quote! conn)
+                        (list {::quotes/id ::quotes/teams-per-profile
+                               ::quotes/profile-id profile-id}
+                              {::quotes/id ::quotes/invitations-per-team
+                               ::quotes/profile-id profile-id
+                               ::quotes/team-id (:id team)
+                               ::quotes/incr (count emails)}
+                              {::quotes/id ::quotes/profiles-per-team
+                               ::quotes/profile-id profile-id
+                               ::quotes/team-id (:id team)
+                               ::quotes/incr (count emails)}))
 
-      (audit/submit! cfg
-                     {::audit/type "command"
-                      ::audit/name "create-team-invitations"
-                      ::audit/profile-id profile-id
-                      ::audit/props {:emails emails
-                                     :role role
-                                     :profile-id profile-id
-                                     :invitations (count emails)}})
-
-      (vary-meta team assoc ::audit/props {:invitations (count emails)}))))
+                  (vary-meta team assoc ::audit/props {:invitations (count emails)})))))
 
 ;; --- Query: get-team-invitation-token
 

backend/src/app/rpc/commands/verify_token.clj

@@ -169,19 +169,15 @@
         ;; if we have logged-in user and it matches the invitation we proceed
         ;; with accepting the invitation and joining the current profile to the
         ;; invited team.
-        (let [context (audit/params->context params)
-              props   {:team-id (:team-id claims)
-                       :role (:role claims)
-                       :invitation-id (:id invitation)}]
+        (let [props {:team-id (:team-id claims)
+                     :role (:role claims)
+                     :invitation-id (:id invitation)}
+              event (-> (audit/event-from-rpc-params params)
+                        (assoc ::audit/name "accept-team-invitation")
+                        (assoc ::audit/props props))]
 
           (accept-invitation cfg claims invitation profile)
-          (audit/submit! cfg
-                         {::audit/type "action"
-                          ::audit/name "accept-team-invitation"
-                          ::audit/profile-id profile-id
-                          ::audit/props props
-                          ::audit/context context})
-
+          (audit/submit! cfg event)
           (assoc claims :state :created))
 
         (ex/raise :type :validation

backend/src/app/rpc/rlimit.clj

@@ -51,12 +51,12 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.http :as-alias http]
-   [app.loggers.audit :refer [parse-client-ip]]
    [app.redis :as rds]
    [app.redis.script :as-alias rscript]
    [app.rpc :as-alias rpc]
    [app.rpc.helpers :as rph]
    [app.rpc.rlimit.result :as-alias lresult]
+   [app.util.inet :as inet]
    [app.util.services :as-alias sv]
    [app.util.time :as dt]
    [app.worker :as wrk]
@@ -215,7 +215,7 @@
   [{:keys [::rpc/profile-id] :as params}]
   (let [request (-> params meta ::http/request)]
     (or profile-id
-        (some-> request parse-client-ip)
+        (some-> request inet/parse-request)
         uuid/zero)))
 
 (defn process-request!

backend/src/app/srepl/main.clj

@@ -21,8 +21,10 @@
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
+   [app.db.sql :as-alias sql]
    [app.features.components-v2 :as feat.comp-v2]
    [app.features.fdata :as feat.fdata]
+   [app.loggers.audit :as audit]
    [app.main :as main]
    [app.msgbus :as mbus]
    [app.rpc.commands.auth :as auth]
@@ -38,10 +40,12 @@
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as wrk]
+   [clojure.java.io :as io]
    [clojure.pprint :refer [print-table]]
    [clojure.stacktrace :as strace]
    [clojure.tools.namespace.repl :as repl]
    [cuerdas.core :as str]
+   [datoteka.fs :as fs]
    [promesa.exec :as px]
    [promesa.exec.semaphore :as ps]
    [promesa.util :as pu]))
@@ -190,6 +194,12 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn notify!
+  "Send flash notifications.
+
+  This method allows send flash notifications to specified target destinations.
+  The message can be a free text or a preconfigured one.
+
+  The destination can be: all, profile-id, team-id, or a coll of them."
   [{:keys [::mbus/msgbus ::db/pool]} & {:keys [dest code message level]
                                         :or {code :generic level :info}
                                         :as params}]
@@ -197,10 +207,6 @@
    ["invalid level %" level]
    (contains? #{:success :error :info :warning} level))
 
-  (dm/verify!
-   ["invalid code: %" code]
-   (contains? #{:generic :upgrade-version} code))
-
   (letfn [(send [dest]
             (l/inf :hint "sending notification" :dest (str dest))
             (let [message {:type :notification
@@ -226,6 +232,9 @@
 
           (resolve-dest [dest]
             (cond
+              (= :all dest)
+              [uuid/zero]
+
               (uuid? dest)
               [dest]
 
@@ -241,14 +250,15 @@
                          (mapcat resolve-dest))
                         dest)
 
-              (and (coll? dest)
-                   (every? coll? dest))
+              (and (vector? dest)
+                   (every? vector? dest))
               (sequence (comp
                          (map vec)
                          (mapcat resolve-dest))
                         dest)
 
-              (vector? dest)
+              (and (vector? dest)
+                   (keyword? (first dest)))
               (let [[op param] dest]
                 (cond
                   (= op :email)
@@ -475,6 +485,27 @@
 ;; DELETE/RESTORE OBJECTS (WITH CASCADE, SOFT)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
+(defn delete-file!
+  "Mark a project for deletion"
+  [file-id]
+  (let [file-id (h/parse-uuid file-id)
+        tnow    (dt/now)]
+
+    (audit/insert! main/system
+                   {::audit/name "delete-file"
+                    ::audit/type "action"
+                    ::audit/profile-id uuid/zero
+                    ::audit/props {:id file-id}
+                    ::audit/context {:triggered-by "srepl"
+                                     :cause "explicit call to delete-file!"}
+                    ::audit/tracked-at tnow})
+    (wrk/invoke! (-> main/system
+                     (assoc ::wrk/task :delete-object)
+                     (assoc ::wrk/params {:object :file
+                                          :deleted-at tnow
+                                          :id file-id})))
+    :deleted))
+
 (defn- restore-file*
   [{:keys [::db/conn]} file-id]
   (db/update! conn :file
@@ -502,20 +533,105 @@
 
   :restored)
 
+(defn restore-file!
+  "Mark a file and all related objects as not deleted"
+  [file-id]
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! main/system
+                (fn [system]
+                  (when-let [file (some-> (db/get* system :file
+                                                   {:id file-id}
+                                                   {::db/remove-deleted false
+                                                    ::sql/columns [:id :name]})
+                                          (files/decode-row))]
+                    (audit/insert! system
+                                   {::audit/name "restore-file"
+                                    ::audit/type "action"
+                                    ::audit/profile-id uuid/zero
+                                    ::audit/props file
+                                    ::audit/context {:triggered-by "srepl"
+                                                     :cause "explicit call to restore-file!"}
+                                    ::audit/tracked-at (dt/now)})
+
+                    (restore-file* system file-id))))))
+
+(defn delete-project!
+  "Mark a project for deletion"
+  [project-id]
+  (let [project-id (h/parse-uuid project-id)
+        tnow       (dt/now)]
+
+    (audit/insert! main/system
+                   {::audit/name "delete-project"
+                    ::audit/type "action"
+                    ::audit/profile-id uuid/zero
+                    ::audit/props {:id project-id}
+                    ::audit/context {:triggered-by "srepl"
+                                     :cause "explicit call to delete-project!"}
+                    ::audit/tracked-at tnow})
+
+    (wrk/invoke! (-> main/system
+                     (assoc ::wrk/task :delete-object)
+                     (assoc ::wrk/params {:object :project
+                                          :deleted-at tnow
+                                          :id project-id})))
+    :deleted))
+
 (defn- restore-project*
   [{:keys [::db/conn] :as cfg} project-id]
-
   (db/update! conn :project
               {:deleted-at nil}
               {:id project-id})
 
   (doseq [{:keys [id]} (db/query conn :file
                                  {:project-id project-id}
-                                 {::db/columns [:id]})]
+                                 {::sql/columns [:id]})]
     (restore-file* cfg id))
 
   :restored)
 
+(defn restore-project!
+  "Mark a project and all related objects as not deleted"
+  [project-id]
+  (let [project-id (h/parse-uuid project-id)]
+    (db/tx-run! main/system
+                (fn [system]
+                  (when-let [project (db/get* system :project
+                                              {:id project-id}
+                                              {::db/remove-deleted false})]
+                    (audit/insert! system
+                                   {::audit/name "restore-project"
+                                    ::audit/type "action"
+                                    ::audit/profile-id uuid/zero
+                                    ::audit/props project
+                                    ::audit/context {:triggered-by "srepl"
+                                                     :cause "explicit call to restore-team!"}
+                                    ::audit/tracked-at (dt/now)})
+
+                    (restore-project* system project-id))))))
+
+(defn delete-team!
+  "Mark a team for deletion"
+  [team-id]
+  (let [team-id (h/parse-uuid team-id)
+        tnow    (dt/now)]
+
+    (audit/insert! main/system
+                   {::audit/name "delete-team"
+                    ::audit/type "action"
+                    ::audit/profile-id uuid/zero
+                    ::audit/props {:id team-id}
+                    ::audit/context {:triggered-by "srepl"
+                                     :cause "explicit call to delete-profile!"}
+                    ::audit/tracked-at tnow})
+
+    (wrk/invoke! (-> main/system
+                     (assoc ::wrk/task :delete-object)
+                     (assoc ::wrk/params {:object :team
+                                          :deleted-at tnow
+                                          :id team-id})))
+    :deleted))
+
 (defn- restore-team*
   [{:keys [::db/conn] :as cfg} team-id]
   (db/update! conn :team
@@ -528,84 +644,127 @@
 
   (doseq [{:keys [id]} (db/query conn :project
                                  {:team-id team-id}
-                                 {::db/columns [:id]})]
+                                 {::sql/columns [:id]})]
     (restore-project* cfg id))
 
   :restored)
 
-(defn- restore-profile*
-  [{:keys [::db/conn] :as cfg} profile-id]
-  (db/update! conn :profile
-              {:deleted-at nil}
-              {:id profile-id})
-
-  (doseq [{:keys [id]} (profile/get-owned-teams conn profile-id)]
-    (restore-team* cfg id))
-
-  :restored)
-
-
-(defn restore-deleted-profile!
-  "Mark a team and all related objects as not deleted"
-  [profile-id]
-  (let [profile-id (h/parse-uuid profile-id)]
-    (db/tx-run! main/system restore-profile* profile-id)))
-
-(defn restore-deleted-team!
+(defn restore-team!
   "Mark a team and all related objects as not deleted"
   [team-id]
   (let [team-id (h/parse-uuid team-id)]
-    (db/tx-run! main/system restore-team* team-id)))
+    (db/tx-run! main/system
+                (fn [system]
+                  (when-let [team (some-> (db/get* system :team
+                                                   {:id team-id}
+                                                   {::db/remove-deleted false})
+                                          (teams/decode-row))]
+                    (audit/insert! system
+                                   {::audit/name "restore-team"
+                                    ::audit/type "action"
+                                    ::audit/profile-id uuid/zero
+                                    ::audit/props team
+                                    ::audit/context {:triggered-by "srepl"
+                                                     :cause "explicit call to restore-team!"}
+                                    ::audit/tracked-at (dt/now)})
 
-(defn restore-deleted-project!
-  "Mark a project and all related objects as not deleted"
-  [project-id]
-  (let [project-id (h/parse-uuid project-id)]
-    (db/tx-run! main/system restore-project* project-id)))
+                    (restore-team* system team-id))))))
 
-(defn restore-deleted-file!
-  "Mark a file and all related objects as not deleted"
-  [file-id]
-  (let [file-id (h/parse-uuid file-id)]
-    (db/tx-run! main/system restore-file* file-id)))
-
-(defn delete-team!
-  "Mark a team for deletion"
-  [team-id]
-  (let [team-id (h/parse-uuid team-id)]
-    (wrk/invoke! (-> main/system
-                     (assoc ::wrk/task :delete-object)
-                     (assoc ::wrk/params {:object :team
-                                          :deleted-at (dt/now)
-                                          :id team-id})))))
 (defn delete-profile!
-  "Mark a profile for deletion"
+  "Mark a profile for deletion."
   [profile-id]
-  (let [profile-id (h/parse-uuid profile-id)]
+  (let [profile-id (h/parse-uuid profile-id)
+        tnow       (dt/now)]
+
+    (audit/insert! main/system
+                   {::audit/name "delete-profile"
+                    ::audit/type "action"
+                    ::audit/profile-id uuid/zero
+                    ::audit/context {:triggered-by "srepl"
+                                     :cause "explicit call to delete-profile!"}
+                    ::audit/tracked-at tnow})
+
     (wrk/invoke! (-> main/system
                      (assoc ::wrk/task :delete-object)
                      (assoc ::wrk/params {:object :profile
-                                          :deleted-at (dt/now)
-                                          :id profile-id})))))
-(defn delete-project!
-  "Mark a project for deletion"
-  [project-id]
-  (let [project-id (h/parse-uuid project-id)]
-    (wrk/invoke! (-> main/system
-                     (assoc ::wrk/task :delete-object)
-                     (assoc ::wrk/params {:object :project
-                                          :deleted-at (dt/now)
-                                          :id project-id})))))
+                                          :deleted-at tnow
+                                          :id profile-id})))
+    :deleted))
 
-(defn delete-file!
-  "Mark a project for deletion"
-  [file-id]
-  (let [file-id (h/parse-uuid file-id)]
-    (wrk/invoke! (-> main/system
-                     (assoc ::wrk/task :delete-object)
-                     (assoc ::wrk/params {:object :file
-                                          :deleted-at (dt/now)
-                                          :id file-id})))))
+(defn restore-profile!
+  "Mark a team and all related objects as not deleted"
+  [profile-id]
+  (let [profile-id (h/parse-uuid profile-id)]
+    (db/tx-run! main/system
+                (fn [system]
+                  (when-let [profile (some-> (db/get* system :profile
+                                                      {:id profile-id}
+                                                      {::db/remove-deleted false})
+                                             (profile/decode-row))]
+                    (audit/insert! system
+                                   {::audit/name "restore-profile"
+                                    ::audit/type "action"
+                                    ::audit/profile-id uuid/zero
+                                    ::audit/props (audit/profile->props profile)
+                                    ::audit/context {:triggered-by "srepl"
+                                                     :cause "explicit call to restore-profile!"}
+                                    ::audit/tracked-at (dt/now)})
+
+                    (db/update! system :profile
+                                {:deleted-at nil}
+                                {:id profile-id}
+                                {::db/return-keys false})
+
+                    (doseq [{:keys [id]} (profile/get-owned-teams system profile-id)]
+                      (restore-team* system id))
+
+                    :restored)))))
+
+(defn delete-profiles-in-bulk!
+  [system path]
+  (letfn [(process-data! [system deleted-at emails]
+            (loop [emails  emails
+                   deleted 0
+                   total   0]
+              (if-let [email (first emails)]
+                (if-let [profile (db/get* system :profile
+                                          {:email (str/lower email)}
+                                          {::db/remove-deleted false})]
+                  (do
+                    (audit/insert! system
+                                   {::audit/name "delete-profile"
+                                    ::audit/type "action"
+                                    ::audit/tracked-at deleted-at
+                                    ::audit/props (audit/profile->props profile)
+                                    ::audit/context {:triggered-by "srepl"
+                                                     :cause "explicit call to delete-profiles-in-bulk!"}})
+                    (wrk/invoke! (-> system
+                                     (assoc ::wrk/task :delete-object)
+                                     (assoc ::wrk/params {:object :profile
+                                                          :deleted-at deleted-at
+                                                          :id (:id profile)})))
+                    (recur (rest emails)
+                           (inc deleted)
+                           (inc total)))
+                  (recur (rest emails)
+                         deleted
+                         (inc total)))
+                {:deleted deleted :total total})))]
+
+    (let [path       (fs/path path)
+          deleted-at (dt/minus (dt/now) cf/deletion-delay)]
+
+      (when-not (fs/exists? path)
+        (throw (ex-info "path does not exists" {:path path})))
+
+      (db/tx-run! system
+                  (fn [system]
+                    (with-open [reader (io/reader path)]
+                      (process-data! system deleted-at (line-seq reader))))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; CASCADE FIXING
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn process-deleted-profiles-cascade
   []

backend/src/app/util/inet.clj

@@ -0,0 +1,37 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.util.inet
+  "INET addr parsing and validation helpers"
+  (:require
+   [cuerdas.core :as str]
+   [ring.request :as rreq])
+  (:import
+   com.google.common.net.InetAddresses
+   java.net.InetAddress))
+
+(defn valid?
+  [s]
+  (InetAddresses/isInetAddress s))
+
+(defn normalize
+  [s]
+  (try
+    (let [addr (InetAddresses/forString s)]
+      (.getHostAddress ^InetAddress addr))
+    (catch Throwable _cause
+      nil)))
+
+(defn parse-request
+  [request]
+  (or (some-> (rreq/get-header request "x-real-ip")
+              (normalize))
+      (some-> (rreq/get-header request "x-forwarded-for")
+              (str/split #"\s*,\s*")
+              (first)
+              (normalize))
+      (some-> (rreq/remote-addr request)
+              (normalize))))

backend/src/app/util/websocket.clj

@@ -11,7 +11,7 @@
    [app.common.logging :as l]
    [app.common.transit :as t]
    [app.common.uuid :as uuid]
-   [app.loggers.audit :refer [parse-client-ip]]
+   [app.util.inet :as inet]
    [app.util.time :as dt]
    [promesa.exec :as px]
    [promesa.exec.csp :as sp]
@@ -84,7 +84,7 @@
         output-ch  (sp/chan :buf output-buff-size)
         hbeat-ch   (sp/chan :buf (sp/sliding-buffer 6))
         close-ch   (sp/chan)
-        ip-addr    (parse-client-ip request)
+        ip-addr    (inet/parse-request request)
         uagent     (rreq/get-header request "user-agent")
         id         (uuid/next)
         state      (atom {})

backend/test/backend_tests/rpc_audit_test.clj

@@ -28,7 +28,8 @@
     ring.request/Request
     (get-header [_ name]
       (case name
-        "x-forwarded-for" "127.0.0.44"))))
+        "x-forwarded-for" "127.0.0.44"
+        "x-real-ip" "127.0.0.43"))))
 
 (t/deftest push-events-1
   (with-redefs [app.config/flags #{:audit-log}]
@@ -46,6 +47,7 @@
                              :profile-id (:id prof)
                              :timestamp (dt/now)
                              :type "action"}]}
+
           params  (with-meta params
                     {:app.http/request http-request})
 

common/src/app/common/files/defaults.cljc

@@ -6,4 +6,4 @@
 
 (ns app.common.files.defaults)
 
-(def version 50)
+(def version 51)

common/src/app/common/files/migrations.cljc

@@ -22,6 +22,7 @@
    [app.common.schema :as sm]
    [app.common.svg :as csvg]
    [app.common.text :as txt]
+   [app.common.types.color :as ctc]
    [app.common.types.component :as ctk]
    [app.common.types.file :as ctf]
    [app.common.types.shape :as cts]
@@ -1004,6 +1005,17 @@
         (update :pages-index update-vals update-container)
         (update :components update-vals update-container))))
 
+(def ^:private valid-color?
+  (sm/lazy-validator ::ctc/color))
+
+(defn migrate-up-51
+  "This migration fixes library invalid colors"
+
+  [data]
+  (let [update-colors
+        (fn [colors]
+          (into {} (filter #(-> % val valid-color?) colors)))]
+    (update data :colors update-colors)))
 
 (def migrations
   "A vector of all applicable migrations"
@@ -1046,4 +1058,5 @@
    {:id 47 :migrate-up migrate-up-47}
    {:id 48 :migrate-up migrate-up-48}
    {:id 49 :migrate-up migrate-up-49}
-   {:id 50 :migrate-up migrate-up-50}])
+   {:id 50 :migrate-up migrate-up-50}
+   {:id 51 :migrate-up migrate-up-51}])

common/src/app/common/types/file.cljc

@@ -633,19 +633,24 @@
   "Find all assets of a library that are used in the file, and
   move them to the file local library."
   [file-data library-data]
-  (let [used-components (find-asset-type-usages file-data library-data :component)
-        used-colors (find-asset-type-usages file-data library-data :color)
-        used-typographies (find-asset-type-usages file-data library-data :typography)]
+  (let [used-components   (find-asset-type-usages file-data library-data :component)
+        file-data         (cond-> file-data
+                            (d/not-empty? used-components)
+                            (absorb-components used-components library-data))
+                            ;; Note that absorbed components may also be using colors
+                            ;; and typographies. This is the reason of doing this first
+                            ;; and accumulating file data for the next ones.
 
-    (cond-> file-data
-      (d/not-empty? used-components)
-      (absorb-components used-components library-data)
+        used-colors       (find-asset-type-usages file-data library-data :color)
+        file-data         (cond-> file-data
+                            (d/not-empty? used-colors)
+                            (absorb-colors used-colors))
 
-      (d/not-empty? used-colors)
-      (absorb-colors used-colors)
-
-      (d/not-empty? used-typographies)
-      (absorb-typographies used-typographies))))
+        used-typographies (find-asset-type-usages file-data library-data :typography)
+        file-data         (cond-> file-data
+                            (d/not-empty? used-typographies)
+                            (absorb-typographies used-typographies))]
+    file-data))
 
 ;; Debug helpers
 

docker/devenv/files/nginx.conf

@@ -68,7 +68,10 @@ http {
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
-        resolver 127.0.0.11;
+        proxy_buffer_size 16k;
+        proxy_busy_buffers_size 24k; # essentially, proxy_buffer_size + 2 small buffers of 4k
+        proxy_buffers 32 4k;
+        resolver 127.0.0.11 ipv6=off;
 
         etag off;
 

frontend/scripts/build

@@ -8,6 +8,7 @@ export CURRENT_VERSION=$1;
 export BUILD_DATE=$(date -R);
 export CURRENT_HASH=${CURRENT_HASH:-$(git rev-parse --short HEAD)};
 export EXTRA_PARAMS=$SHADOWCLJS_EXTRA_PARAMS;
+export TS=$(date +%s);
 
 # Some cljs reacts on this environment variable for define more
 # performant code on macros (example: rumext)
@@ -17,7 +18,7 @@ yarn install || exit 1;
 rm -rf resources/public;
 rm -rf target/dist;
 
-clojure -M:dev:shadow-cljs release main --config-merge "{:release-version \"${CURRENT_HASH}\"}" $EXTRA_PARAMS || exit 1
+clojure -M:dev:shadow-cljs release main --config-merge "{:release-version \"${CURRENT_HASH}-${TS}\"}" $EXTRA_PARAMS || exit 1
 
 yarn run compile || exit 1;
 mkdir -p target/dist;

frontend/src/app/main/data/common.cljs

@@ -13,6 +13,7 @@
    [app.main.data.modal :as modal]
    [app.main.features :as features]
    [app.main.repo :as rp]
+   [app.main.store :as st]
    [app.util.i18n :refer [tr]]
    [beicon.v2.core :as rx]
    [potok.v2.core :as ptk]))
@@ -58,6 +59,10 @@
   []
   (.reload js/location))
 
+(defn hide-notifications!
+  []
+  (st/emit! msg/hide))
+
 (defn handle-notification
   [{:keys [message code level] :as params}]
   (ptk/reify ::show-notification
@@ -75,6 +80,15 @@
                   :actions [{:label "Refresh" :callback force-reload!}]
                   :tag :notification)))
 
+        :maintenance
+        (rx/of (msg/dialog
+                :content (tr "notifications.by-code.maintenance")
+                :controls :inline-actions
+                :type level
+                :actions [{:label (tr "labels.accept")
+                           :callback hide-notifications!}]
+                :tag :notification))
+
         (rx/of (msg/dialog
                 :content message
                 :controls :close

frontend/src/app/main/data/messages.cljs

@@ -15,42 +15,42 @@
 (declare hide)
 (declare show)
 
-(def default-animation-timeout 600)
 (def default-timeout 7000)
 
-(def ^:private
-  schema:message
-  (sm/define
-    [:map {:title "Message"}
-     [:type [::sm/one-of #{:success :error :info :warning}]]
-     [:status {:optional true}
-      [::sm/one-of #{:visible :hide}]]
-     [:position {:optional true}
-      [::sm/one-of #{:fixed :floating :inline}]]
-     [:notification-type {:optional true}
-      [::sm/one-of #{:inline :context :toast}]]
-     [:controls {:optional true}
-      [::sm/one-of #{:none :close :inline-actions :bottom-actions}]]
-     [:tag {:optional true}
-      [:or :string :keyword]]
-     [:timeout {:optional true}
-      [:maybe :int]]
-     [:actions {:optional true}
-      [:vector
-       [:map
-        [:label :string]
-        [:callback ::sm/fn]]]]
-     [:links {:optional true}
-      [:vector
-       [:map
-        [:label :string]
-        [:callback ::sm/fn]]]]]))
+(def ^:private schema:message
+  [:map {:title "Message"}
+   [:type [::sm/one-of #{:success :error :info :warning}]]
+   [:status {:optional true}
+    [::sm/one-of #{:visible :hide}]]
+   [:position {:optional true}
+    [::sm/one-of #{:fixed :floating :inline}]]
+   [:notification-type {:optional true}
+    [::sm/one-of #{:inline :context :toast}]]
+   [:controls {:optional true}
+    [::sm/one-of #{:none :close :inline-actions :bottom-actions}]]
+   [:tag {:optional true}
+    [:or :string :keyword]]
+   [:timeout {:optional true}
+    [:maybe :int]]
+   [:actions {:optional true}
+    [:vector
+     [:map
+      [:label :string]
+      [:callback ::sm/fn]]]]
+   [:links {:optional true}
+    [:vector
+     [:map
+      [:label :string]
+      [:callback ::sm/fn]]]]])
+
+(def ^:private valid-message?
+  (sm/validator schema:message))
 
 (defn show
   [data]
   (dm/assert!
    "expected valid message map"
-   (sm/check! schema:message data))
+   (valid-message? data))
 
   (ptk/reify ::show
     ptk/UpdateEvent
@@ -76,14 +76,7 @@
   (ptk/reify ::hide
     ptk/UpdateEvent
     (update [_ state]
-      (d/update-when state :message assoc :status :hide))
-
-    ptk/WatchEvent
-    (watch [_ _ stream]
-      (let [stopper (rx/filter (ptk/type? ::show) stream)]
-        (->> (rx/of #(dissoc % :message))
-             (rx/delay default-animation-timeout)
-             (rx/take-until stopper))))))
+      (dissoc state :message))))
 
 (defn hide-tag
   [tag]

frontend/src/app/main/data/users.cljs

@@ -696,15 +696,20 @@
   (ptk/reify ::show-redirect-error
     ptk/WatchEvent
     (watch [_ _ _]
-      (let [hint (case error
-                   "registration-disabled"
-                   (tr "errors.registration-disabled")
-                   "profile-blocked"
-                   (tr "errors.profile-blocked")
-                   "auth-provider-not-allowed"
-                   (tr "errors.auth-provider-not-allowed")
-                   "email-domain-not-allowed"
-                   (tr "errors.email-domain-not-allowed")
-                   :else
-                   (tr "errors.generic"))]
+      (when-let [hint (case error
+                        "registration-disabled"
+                        (tr "errors.registration-disabled")
+                        "profile-blocked"
+                        (tr "errors.profile-blocked")
+                        "auth-provider-not-allowed"
+                        (tr "errors.auth-provider-not-allowed")
+                        "email-domain-not-allowed"
+                        (tr "errors.email-domain-not-allowed")
+
+                        ;; We explicitly do not show any error here, it a explicit user operation.
+                        "unable-to-auth"
+                        nil
+
+                        (tr "errors.generic"))]
+
         (rx/of (msg/warn hint))))))

frontend/src/app/main/ui/components/radio_buttons.cljs

@@ -58,7 +58,7 @@
 
 (mf/defc radio-buttons
   {::mf/props :obj}
-  [{:keys [children on-change selected class wide encode-fn decode-fn allow-empty] :as props}]
+  [{:keys [name children on-change selected class wide encode-fn decode-fn allow-empty] :as props}]
   (let [encode-fn (d/nilv encode-fn identity)
         decode-fn (d/nilv decode-fn identity)
         nitems    (if (array? children)
@@ -94,5 +94,6 @@
 
     [:& (mf/provider context) {:value context-value}
      [:div {:class (dm/str class " " (stl/css :radio-btn-wrapper))
-            :style {:width width}}
+            :style {:width width}
+            :key (dm/str name "-" selected)}
       children]]))

frontend/src/app/main/ui/dashboard/placeholder.cljs

@@ -41,5 +41,5 @@
 (mf/defc loading-placeholder
   []
   [:div {:class (stl/css :grid-empty-placeholder :loader)}
-   [:div {:class (stl/css :icon)} i/loader]
+   [:div {:class (stl/css :icon)} i/loader-pencil]
    [:div {:class (stl/css :text)} (tr "dashboard.loading-files")]])

frontend/src/app/main/ui/dashboard/placeholder.scss

@@ -23,8 +23,7 @@
     svg {
       width: $s-64;
       height: $s-64;
-      stroke: $df-secondary;
-      fill: none;
+      fill: $df-secondary;
     }
   }
 

frontend/src/app/main/ui/messages.cljs

@@ -17,33 +17,38 @@
 (mf/defc notifications-hub
   []
   (let [message  (mf/deref refs/message)
-
-        on-close #(st/emit! dmsg/hide)
-
-        toast-message   {:type (or (:type message) :info)
-                         :links (:links message)
-                         :on-close on-close
-                         :content (:content message)}
-
-        inline-message  {:actions (:actions message)
-                         :links (:links message)
-                         :content (:content message)}
-
-        context-message {:type (or (:type message) :info)
-                         :links (:links message)
-                         :content (:content message)}
-
-        is-context-msg (and (nil? (:timeout message)) (nil? (:actions message)))
-        is-toast-msg   (or (= :toast (:notification-type message)) (some? (:timeout message)))
-        is-inline-msg  (or (= :inline (:notification-type message)) (and (some? (:position message)) (= :floating (:position message))))]
+        on-close (mf/use-fn #(st/emit! dmsg/hide))
+        context? (and (nil? (:timeout message))
+                      (nil? (:actions message)))
+        inline?  (or (= :inline (:notification-type message))
+                     (= :floating (:position message)))
+        toast?   (or (= :toast (:notification-type message))
+                     (some? (:timeout message)))]
 
     (when message
       (cond
-        is-toast-msg
-        [:& toast-notification toast-message]
-        is-inline-msg
-        [:& inline-notification inline-message]
-        is-context-msg
-        [:& context-notification context-message]
+        toast?
+        [:& toast-notification
+         {:type (or (:type message) :info)
+          :links (:links message)
+          :on-close on-close
+          :content (:content message)}]
+
+        inline?
+        [:& inline-notification
+         {:actions (:actions message)
+          :links (:links message)
+          :content (:content message)}]
+
+        context?
+        [:& context-notification
+         {:type (or (:type message) :info)
+          :links (:links message)
+          :content (:content message)}]
+
         :else
-        [:& toast-notification toast-message]))))
+        [:& toast-notification
+         {:type (or (:type message) :info)
+          :links (:links message)
+          :on-close on-close
+          :content (:content message)}]))))

frontend/src/app/main/ui/notifications/toast_notification.cljs

@@ -38,12 +38,10 @@
     neutral-icon))
 
 (mf/defc toast-notification
-  "These are ephemeral elements that disappear when
-   the close button is pressed,
-   the page is refreshed,
-   the page is navigated to another page or
-   after 7 seconds, which is enough time to be read,
-   except for error messages that require user interaction."
+  "These are ephemeral elements that disappear when the close button
+  is pressed, the page is refreshed, the page is navigated to another
+  page or after 7 seconds, which is enough time to be read, except for
+  error messages that require user interaction."
 
   {::mf/props :obj}
   [{:keys [type content on-close links] :as props}]

frontend/src/app/main/ui/onboarding/questions.cljs

@@ -411,9 +411,9 @@
         (mf/with-memo []
           (-> (shuffle [{:label (tr "labels.youtube") :value "youtube"}
                         {:label (tr "labels.event") :value "event"}
-                        {:label (tr "labels.search") :value "search"}
-                        {:label (tr "labels.social") :value "social"}
-                        {:label (tr "labels.article") :value "article"}])
+                        {:label (tr "onboarding.questions.referer.search") :value "search"}
+                        {:label (tr "onboarding.questions.referer.social") :value "social"}
+                        {:label (tr "onboarding.questions.referer.article") :value "article"}])
               (conj {:label (tr "labels.other-short") :value "other"})))
 
         current-referer

frontend/src/app/main/ui/workspace/sidebar/assets/colors.cljs

@@ -264,8 +264,10 @@
            multi-colors? multi-assets? on-asset-click on-assets-delete
            on-clear-selection on-group on-rename-group on-ungroup colors
            selected-full]}]
-  (let [group-open?    (or ^boolean force-open?
-                           ^boolean (get open-groups prefix (if (= prefix "") true false)))
+  (let [group-open?    (if (false? (get open-groups prefix)) ;; if the user has closed it specifically, respect that
+                         false
+                         (or ^boolean force-open?
+                             ^boolean (get open-groups prefix (if (= prefix "") true false))))
         dragging*      (mf/use-state false)
         dragging?      (deref dragging*)
 

frontend/src/app/main/ui/workspace/sidebar/assets/typographies.cljs

@@ -128,7 +128,9 @@
   [{:keys [file-id prefix groups open-groups force-open? file local? selected local-data
            editing-id renaming-id on-asset-click handle-change apply-typography on-rename-group
            on-ungroup on-context-menu selected-full]}]
-  (let [group-open?   (get open-groups prefix true)
+  (let [group-open?    (if (false? (get open-groups prefix)) ;; if the user has closed it specifically, respect that
+                         false
+                         (get open-groups prefix true))
         dragging*      (mf/use-state false)
         dragging?      (deref dragging*)
         selected-paths (mf/with-memo [selected-full]

frontend/translations/en.po

@@ -2209,6 +2209,10 @@ msgstr "Update a component in a shared library"
 msgid "notifications.by-code.upgrade-version"
 msgstr "A new version is available, please refresh the page"
 
+#: src/app/main/data/common.cljs
+msgid "notifications.by-code.maintenance"
+msgstr "Maintenance break: we will be down for a short maintenance within 5 minutes."
+
 #: src/app/main/ui/dashboard/team.cljs
 msgid "notifications.invitation-email-sent"
 msgstr "Invitation sent successfully"
@@ -2577,15 +2581,15 @@ msgid "labels.event"
 msgstr "Event"
 
 #: src/app/main/ui/onboarding/questions.cljs
-msgid "labels.search"
+msgid "onboarding.questions.referer.search"
 msgstr "Search Engine (Google, Yahoo, Bing)"
 
 #: src/app/main/ui/onboarding/questions.cljs
-msgid "labels.social"
+msgid "onboarding.questions.referer.social"
 msgstr "Social Media (X, Linkedin, FB, etc)"
 
 #: src/app/main/ui/onboarding/questions.cljs
-msgid "labels.article"
+msgid "onboarding.questions.referer.article"
 msgstr "Article (Blog, Post, Newsletter)"
 
 #: src/app/main/ui/onboarding/questions.cljs

frontend/translations/es.po

@@ -2285,6 +2285,10 @@ msgstr "Actualizar un componente en biblioteca"
 msgid "notifications.by-code.upgrade-version"
 msgstr "Una nueva versión está disponible, por favor actualiza la página"
 
+#: src/app/main/data/common.cljs
+msgid "notifications.by-code.maintenance"
+msgstr "Pausa de mantenimiento: en los próximos 5 minutos estaremos fuera de servicio por un breve mantenimiento."
+
 #: src/app/main/ui/dashboard/team.cljs
 msgid "notifications.invitation-email-sent"
 msgstr "Invitación enviada con éxito"