🐛 Fix problem with tokens in plugins

common/src/app/common/data.cljc

@@ -760,17 +760,6 @@
        default
        v))))
 
-(defn parse-percent
-  ([v]
-   (parse-percent v nil))
-  ([v default]
-  (if (str/ends-with? v "%")
-    (let [v (impl-parse-double (str/replace v "%" ""))]
-      (if (or (nil? v) (nan? v))
-        default
-        (/ v 100)))
-    (parse-double v default))))
-
 (defn parse-uuid
   [v]
   (try

common/src/app/common/files/tokens.cljc

@@ -31,49 +31,18 @@
 (def schema:token-value-generic
   [::sm/text {:error/fn token-value-empty-fn}])
 
-(def schema:token-value-numeric
-  [:and
-   [::sm/text {:error/fn token-value-empty-fn}]
-   [:or
-    cto/schema:token-ref
-    [:fn {:error/fn #(tr "workspace.tokens.invalid-value")}
-     (fn [value]
-       (let [n (d/parse-double value)]
-         (some? n)))]]])
-
-(def schema:token-value-percent
-  [:and
-   [::sm/text {:error/fn token-value-empty-fn}]
-   [:or cto/schema:token-ref
-    [:fn {:error/fn #(tr "workspace.tokens.value-with-percent")}
-     (fn [value]
-       (let [v (d/parse-percent value)]
-         (some? v)))]]])
-
 (def schema:token-value-composite-ref
   [::sm/text {:error/fn token-value-empty-fn}])
 
-(def schema:token-value-opacity
-  [:and
-   ::sm/text
-   [:fn {:error/fn #(tr "workspace.tokens.opacity-range")}
-    (fn [opacity]
-      (let [n (d/parse-percent opacity)]
-        (and (some? n) (<= 0 n 1))))]])
-
 (def schema:token-value-font-family
   [:vector ::sm/text])
 
-(def schema:token-value-font-weight
-  [:fn {:error/fn #(tr "workspace.tokens.invalid-font-weight-token-value")}
-   cto/valid-font-weight-variant])
-
 (def schema:token-value-typography-map
   [:map
    [:font-family {:optional true} schema:token-value-font-family]
-   [:font-size {:optional true} schema:token-value-numeric]
-   [:font-weight {:optional true} schema:token-value-font-weight]
-   [:line-height {:optional true} schema:token-value-percent]
+   [:font-weight {:optional true} schema:token-value-generic]
+   [:font-size {:optional true} schema:token-value-generic]
+   [:line-height {:optional true} schema:token-value-generic]
    [:letter-spacing {:optional true} schema:token-value-generic]
    [:paragraph-spacing {:optional true} schema:token-value-generic]
    [:text-decoration {:optional true} schema:token-value-generic]
@@ -115,10 +84,7 @@
   [token-type]
   [:multi {:dispatch (constantly token-type)
            :title "Token Value"}
-   [:opacity schema:token-value-opacity]
    [:font-family schema:token-value-font-family]
-   [:font-size schema:token-value-numeric]
-   [:font-weight schema:token-value-font-weight]
    [:typography schema:token-value-typography]
    [:shadow schema:token-value-shadow]
    [::m/default schema:token-value-generic]])

common/src/app/common/types/token.cljc

@@ -143,15 +143,6 @@
         :gen/gen sg/text}
    token-name-validation-regex])
 
-(def token-ref-validation-regex
-  #"^\{[a-zA-Z0-9_-][a-zA-Z0-9$_-]*(\.[a-zA-Z0-9$_-]+)*\}$")
-
-(def schema:token-ref
-  "A token reference is a token name enclosed in {}."
-  [:re {:title "TokenRef"
-        :gen/gen sg/text}
-   token-ref-validation-regex])
-
 (def schema:token-type
   [::sm/one-of {:decode/json (fn [type]
                                (if (string? type)

docker/devenv/docker-compose.yaml

@@ -50,6 +50,7 @@ services:
       - 4400:4400
       - 4401:4401
       - 4402:4402
+      - 4403:4403
 
       # Plugins
       - 4200:4200

frontend/src/app/main/data/tokenscript.cljs

@@ -69,10 +69,6 @@
   (and (number-with-unit-symbol? v)
        (= (.-unit v) "rem")))
 
-(defn percent-number-with-unit? [v]
-  (and (number-with-unit-symbol? v)
-       (= (.-unit v) "%")))
-
 (defn rem->px [^js v]
   (* (.-value v) 16))
 
@@ -91,12 +87,10 @@
 
 (defn tokenscript-symbols->penpot-unit [^js v]
   (cond
-    (nil? v) nil
     (structured-token? v) (structured-token->penpot-map v)
     (list-symbol? v) (tokenscript-symbols->penpot-unit (.nth 1 v))
     (color-symbol? v) (.-value (.to v "hex"))
     (rem-number-with-unit? v) (rem->px v)
-    (percent-number-with-unit? v) (/ (.-value v) 100)
     :else (.-value v)))
 
 ;; Processors ------------------------------------------------------------------

frontend/src/app/main/data/workspace/tokens/import_export.cljs

@@ -8,11 +8,10 @@
   (:require
    [app.common.json :as json]
    [app.common.path-names :as cpn]
+
    [app.common.types.tokens-lib :as ctob]
-   [app.config :as cf]
    [app.main.data.notifications :as ntf]
    [app.main.data.style-dictionary :as sd]
-   [app.main.data.tokenscript :as ts]
    [app.main.data.workspace.tokens.errors :as wte]
    [app.main.store :as st]
    [app.util.i18n :refer [tr]]
@@ -75,18 +74,15 @@
   (when unknown-tokens
     (st/emit! (show-unknown-types-warning unknown-tokens)))
   (try
-    (let [tokens-tree     (ctob/get-all-tokens-map tokens-lib)
-          resolved-tokens (if (contains? cf/flags :tokenscript)
-                            (rx/of (ts/resolve-tokens tokens-tree))
-                            (sd/resolve-tokens-with-verbose-errors tokens-tree))]
-      (->> resolved-tokens
-           (rx/map (fn [_]
-                     tokens-lib))
-           (rx/catch (fn [sd-error]
-                       (let [reference-errors (extract-reference-errors sd-error)]
-                         (if reference-errors
-                           (rx/of tokens-lib)
-                           (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error))))))))
+    (->> (ctob/get-all-tokens-map tokens-lib)
+         (sd/resolve-tokens-with-verbose-errors)
+         (rx/map (fn [_]
+                   tokens-lib))
+         (rx/catch (fn [sd-error]
+                     (let [reference-errors (extract-reference-errors sd-error)]
+                       (if reference-errors
+                         (rx/of tokens-lib)
+                         (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error sd-error sd-error)))))))
     (catch js/Error e
       (throw (wte/error-ex-info :error.import/style-dictionary-unknown-error "" e)))))
 

frontend/src/app/main/data/workspace/tokens/propagation.cljs

@@ -6,16 +6,13 @@
 
 (ns app.main.data.workspace.tokens.propagation
   (:require
-   [app.common.data :as d]
    [app.common.files.helpers :as cfh]
    [app.common.logging :as l]
    [app.common.time :as ct]
    [app.common.types.token :as ctt]
    [app.common.types.tokens-lib :as ctob]
-   [app.config :as cf]
    [app.main.data.helpers :as dsh]
    [app.main.data.style-dictionary :as sd]
-   [app.main.data.tokenscript :as ts]
    [app.main.data.workspace.shapes :as dwsh]
    [app.main.data.workspace.thumbnails :as dwt]
    [app.main.data.workspace.tokens.application :as dwta]
@@ -213,13 +210,10 @@
   (ptk/reify ::propagate-workspace-tokens
     ptk/WatchEvent
     (watch [_ state _]
-      (when-let [tokens-tree (-> (dsh/lookup-file-data state)
-                                 (get :tokens-lib)
-                                 (ctob/get-tokens-in-active-sets))]
-        (->> (if (contains? cf/flags :tokenscript)
-               (rx/of (-> (ts/resolve-tokens tokens-tree)
-                          (d/update-vals #(update % :resolved-value ts/tokenscript-symbols->penpot-unit))))
-               (sd/resolve-tokens tokens-tree))
+      (when-let [tokens-lib (-> (dsh/lookup-file-data state)
+                                (get :tokens-lib))]
+        (->> (ctob/get-tokens-in-active-sets tokens-lib)
+             (sd/resolve-tokens)
              (rx/mapcat (fn [sd-tokens]
                           (let [undo-id (js/Symbol)]
                             (rx/concat

frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs

@@ -13,10 +13,8 @@
    [app.common.types.color :as cl]
    [app.common.types.token :as cto]
    [app.common.types.tokens-lib :as ctob]
-   [app.config :as cf]
    [app.main.data.style-dictionary :as sd]
    [app.main.data.tinycolor :as tinycolor]
-   [app.main.data.tokenscript :as ts]
    [app.main.data.workspace.tokens.format :as dwtf]
    [app.main.refs :as refs]
    [app.main.ui.ds.controls.input :as ds]
@@ -72,15 +70,11 @@
             (dissoc (:name prev-token))
             (update (:name token) #(ctob/make-token (merge % prev-token token))))]
 
-    (->> (if (contains? cf/flags :tokenscript)
-           (rx/of (ts/resolve-tokens tokens))
-           (sd/resolve-tokens-interactive tokens))
+    (->> tokens
+         (sd/resolve-tokens-interactive)
          (rx/mapcat
           (fn [resolved-tokens]
-            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))
-                  resolved-value (if (contains? cf/flags :tokenscript)
-                                   (ts/tokenscript-symbols->penpot-unit resolved-value)
-                                   resolved-value)]
+            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))]
               (if resolved-value
                 (rx/of {:value resolved-value})
                 (rx/of {:error (first errors)}))))))))
@@ -238,7 +232,6 @@
                                  (tinycolor/set-alpha (or alpha 1))
                                  (tinycolor/->string format))]
              (when (not= value color-value)
-               (fm/on-input-change form input-name color-value true)
                (fm/on-input-change form input-name color-value true)
                (rx/push! resolve-stream color-value)))))
 

frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs

@@ -10,9 +10,7 @@
    [app.common.data :as d]
    [app.common.types.token :as cto]
    [app.common.types.tokens-lib :as ctob]
-   [app.config :as cf]
    [app.main.data.style-dictionary :as sd]
-   [app.main.data.tokenscript :as ts]
    [app.main.fonts :as fonts]
    [app.main.ui.ds.buttons.icon-button :refer [icon-button*]]
    [app.main.ui.ds.controls.input :refer [input*]]
@@ -68,15 +66,11 @@
             (dissoc (:name prev-token))
             (update (:name token) #(ctob/make-token (merge % prev-token token))))]
 
-    (->> (if (contains? cf/flags :tokenscript)
-           (rx/of (ts/resolve-tokens tokens))
-           (sd/resolve-tokens-interactive tokens))
+    (->> tokens
+         (sd/resolve-tokens-interactive)
          (rx/mapcat
           (fn [resolved-tokens]
-            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))
-                  resolved-value (if (contains? cf/flags :tokenscript)
-                                   (ts/tokenscript-symbols->penpot-unit resolved-value)
-                                   resolved-value)]
+            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))]
               (if resolved-value
                 (rx/of {:value resolved-value})
                 (rx/of {:error (first errors)}))))))))

frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs

@@ -175,10 +175,7 @@
          (sd/resolve-tokens-interactive)
          (rx/mapcat
           (fn [resolved-tokens]
-            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))
-                  resolved-value (if (contains? cf/flags :tokenscript)
-                                   (ts/tokenscript-symbols->penpot-unit resolved-value)
-                                   resolved-value)]
+            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))]
               (if resolved-value
                 (rx/of {:value resolved-value})
                 (rx/of {:error (first errors)}))))))))

frontend/src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs

@@ -36,9 +36,9 @@
    [rumext.v2 :as mf]))
 
 (defn get-value-for-validator
-  [active-tab value value-subfield value-type]
+  [active-tab value value-subfield form-type]
 
-  (case value-type
+  (case form-type
     :indexed
     (if (= active-tab :reference)
       (:reference value)
@@ -62,7 +62,7 @@
            make-schema
            input-component
            initial
-           value-type
+           type
            value-subfield
            input-value-placeholder] :as props}]
 
@@ -178,13 +178,13 @@
 
         on-submit
         (mf/use-fn
-         (mf/deps validate-token token tokens token-type value-subfield value-type active-tab on-remap-token on-rename-token is-create)
+         (mf/deps validate-token token tokens token-type value-subfield type active-tab on-remap-token on-rename-token is-create)
          (fn [form _event]
            (let [name (get-in @form [:clean-data :name])
                  path (str (d/name token-type) "." name)
                  description (get-in @form [:clean-data :description])
                  value (get-in @form [:clean-data :value])
-                 value-for-validation (get-value-for-validator active-tab value value-subfield value-type)]
+                 value-for-validation (get-value-for-validator active-tab value value-subfield type)]
              (->> (validate-token {:token-value value-for-validation
                                    :token-name name
                                    :token-description description
@@ -245,7 +245,7 @@
                            :auto-focus true}]]
 
       [:div {:class (stl/css :input-row)}
-       (case value-type
+       (case type
          :indexed
          [:> input-component
           {:token          token

frontend/src/app/main/ui/workspace/tokens/management/forms/shadow.cljs

@@ -365,7 +365,7 @@
                                       :token-type token-type
                                       :initial initial
                                       :make-schema make-schema
-                                      :value-type :indexed
+                                      :type :indexed
                                       :value-subfield :shadow
                                       :input-component tabs-wrapper*
                                       :validator validate-shadow-token})]

frontend/src/app/main/ui/workspace/tokens/management/forms/typography.cljs

@@ -300,6 +300,6 @@
                                       :make-schema make-schema
                                       :token token
                                       :validator validate-typography-token
-                                      :value-type :composite
+                                      :type :composite
                                       :input-component tabs-wrapper*})]
     [:> generic/form* props]))

frontend/src/app/main/ui/workspace/tokens/management/forms/validators.cljs

@@ -4,9 +4,7 @@
    [app.common.schema :as sm]
    [app.common.types.token :as cto]
    [app.common.types.tokens-lib :as ctob]
-   [app.config :as cf]
    [app.main.data.style-dictionary :as sd]
-   [app.main.data.tokenscript :as ts]
    [app.main.data.workspace.tokens.errors :as wte]
    [beicon.v2.core :as rx]
    [cuerdas.core :as str]))
@@ -38,20 +36,14 @@
 
                   :always
                   (update (:name token) #(ctob/make-token (merge % prev-token token))))]
-
-    (->> (if (contains? cf/flags :tokenscript)
-           (rx/of (ts/resolve-tokens tokens'))
-           (sd/resolve-tokens-interactive tokens'))
+    (->> tokens'
+         (sd/resolve-tokens-interactive)
          (rx/mapcat
           (fn [resolved-tokens]
-            (let [resolved-token (cond-> (get resolved-tokens (:name token))
-                                   (contains? cf/flags :tokenscript)
-                                   (update :resolved-value ts/tokenscript-symbols->penpot-unit))]
+            (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))]
               (cond
-                (:resolved-value resolved-token)
-                (rx/of resolved-token)
-
-                :else (rx/throw {:errors (or (seq (:errors resolved-token))
+                resolved-value (rx/of resolved-token)
+                :else (rx/throw {:errors (or (seq errors)
                                              [(wte/get-error-code :error/unknown-error)])}))))))))
 
 (defn- validate-token-with [token validators]

frontend/src/app/plugins/shape.cljs

@@ -1305,7 +1305,8 @@
                  tokens)))}
 
            :applyToken
-           {:schema [:tuple
+           {:enumerable false
+            :schema [:tuple
                      [:fn token-proxy?]
                      [:maybe [:set [:and ::sm/keyword [:fn cto/token-attr?]]]]]
             :fn (fn [token attrs]

frontend/src/app/plugins/tokens.cljs

@@ -12,12 +12,14 @@
    [app.common.types.token :as cto]
    [app.common.types.tokens-lib :as ctob]
    [app.common.uuid :as uuid]
+   [app.main.data.style-dictionary :as sd]
    [app.main.data.tokenscript :as ts]
    [app.main.data.workspace.tokens.application :as dwta]
    [app.main.data.workspace.tokens.library-edit :as dwtl]
    [app.main.store :as st]
    [app.plugins.utils :as u]
    [app.util.object :as obj]
+   [beicon.v2.core :as rx]
    [clojure.datafy :refer [datafy]]))
 
 ;; === Token
@@ -142,14 +144,16 @@
       (st/emit! (dwtl/delete-token set-id id)))
 
     :applyToShapes
-    {:schema [:tuple
+    {:enumerable false
+     :schema [:tuple
               [:vector [:fn shape-proxy?]]
               [:maybe [:set [:and ::sm/keyword [:fn cto/token-attr?]]]]]
      :fn (fn [shapes attrs]
            (apply-token-to-shapes file-id set-id id (map #(obj/get % "$id") shapes) attrs))}
 
     :applyToSelected
-    {:schema [:tuple [:maybe [:set [:and ::sm/keyword [:fn cto/token-attr?]]]]]
+    {:enumerable false
+     :schema [:tuple [:maybe [:set [:and ::sm/keyword [:fn cto/token-attr?]]]]]
      :fn (fn [attrs]
            (let [selected (get-in @st/state [:workspace-local :selected])]
              (apply-token-to-shapes file-id set-id id selected attrs)))}))
@@ -234,14 +238,16 @@
               (apply array))))}
 
     :getTokenById
-    {:schema [:tuple ::sm/uuid]
+    {:enumerable false
+     :schema [:tuple ::sm/uuid]
      :fn (fn [token-id]
            (let [token (u/locate-token file-id id token-id)]
              (when (some? token)
                (token-proxy plugin-id file-id id token-id))))}
 
     :addToken
-    {:schema (fn [args]
+    {:enumerable false
+     :schema (fn [args]
                [:tuple (-> (cfo/make-token-schema
                             (-> (u/locate-tokens-lib file-id) (ctob/get-tokens id))
                             (cto/dtcg-token-type->token-type (-> args (first) (get "type"))))
@@ -254,19 +260,20 @@
      :decode/options {:key-fn identity}
      :fn (fn [attrs]
            (let [tokens-lib (u/locate-tokens-lib file-id)
-                 token (ctob/make-token attrs)
-                 tokens-tree (-> (ctob/get-tokens-in-active-sets tokens-lib)
-                                 (assoc (:name token) token))
-                 resolved-tokens (ts/resolve-tokens tokens-tree)
-
-                 {:keys [errors resolved-value] :as resolved-token}
-                 (get resolved-tokens (:name token))]
-
-             (if resolved-value
-               (do (st/emit! (dwtl/create-token id token))
-                   (token-proxy plugin-id file-id id (:id token)))
-               (do (u/display-not-valid :addToken (str errors))
-                   nil))))}
+                 tokens-tree (ctob/get-tokens-in-active-sets tokens-lib)
+                 token (ctob/make-token attrs)]
+             (->> (assoc tokens-tree (:name token) token)
+                  (sd/resolve-tokens-interactive)
+                  (rx/subs!
+                   (fn [resolved-tokens]
+                     (let [{:keys [errors resolved-value] :as resolved-token} (get resolved-tokens (:name token))]
+                       (if resolved-value
+                         (st/emit! (dwtl/create-token id token))
+                         (u/display-not-valid :addToken (str errors)))))))
+             ;; TODO: as the addToken function is synchronous, we must return the newly created
+             ;;       token even if the validator will throw it away if the resolution fails.
+             ;;       This will be solved with the TokenScript resolver, that is syncronous.
+             (token-proxy plugin-id file-id id (:id token))))}
 
     :duplicate
     (fn []
@@ -347,26 +354,18 @@
       (st/emit! (dwtl/toggle-token-theme-active id)))
 
     :activeSets
-    {:this true
-     :get (fn [_]
-            (let [tokens-lib (u/locate-tokens-lib file-id)
-                  theme (u/locate-token-theme file-id id)]
-              (->> theme
-                   :sets
-                   (map #(->> %
-                              (ctob/get-set-by-name tokens-lib)
-                              (ctob/get-id)
-                              (token-set-proxy plugin-id file-id)))
-                   (apply array))))}
+    {:this true :get (fn [_])}
 
     :addSet
-    {:schema [:tuple [:fn token-set-proxy?]]
+    {:enumerable false
+     :schema [:tuple [:fn token-set-proxy?]]
      :fn (fn [token-set]
            (let [theme (u/locate-token-theme file-id id)]
              (st/emit! (dwtl/update-token-theme id (ctob/enable-set theme (obj/get token-set :name))))))}
 
     :removeSet
-    {:schema [:tuple [:fn token-set-proxy?]]
+    {:enumerable false
+     :schema [:tuple [:fn token-set-proxy?]]
      :fn (fn [token-set]
            (let [theme (u/locate-token-theme file-id id)]
              (st/emit! (dwtl/update-token-theme id (ctob/disable-set theme (obj/get token-set :name))))))}
@@ -413,7 +412,8 @@
          (apply array (map #(token-set-proxy plugin-id file-id (ctob/get-id %)) sets))))}
 
     :addTheme
-    {:schema (fn [attrs]
+    {:enumerable false
+     :schema (fn [attrs]
                [:tuple (-> (sm/schema (cfo/make-token-theme-schema
                                        (u/locate-tokens-lib file-id)
                                        (or (obj/get attrs "group") "")
@@ -426,7 +426,8 @@
              (token-theme-proxy plugin-id file-id (:id theme))))}
 
     :addSet
-    {:schema [:tuple (-> (sm/schema (cfo/make-token-set-schema
+    {:enumerable false
+     :schema [:tuple (-> (sm/schema (cfo/make-token-set-schema
                                      (u/locate-tokens-lib file-id)
                                      nil))
                          (sm/dissoc-key :id))] ;; We don't allow plugins to set the id
@@ -438,14 +439,16 @@
              (token-set-proxy plugin-id file-id (ctob/get-id set))))}
 
     :getThemeById
-    {:schema [:tuple ::sm/uuid]
+    {:enumerable false
+     :schema [:tuple ::sm/uuid]
      :fn (fn [theme-id]
            (let [theme (u/locate-token-theme file-id theme-id)]
              (when (some? theme)
                (token-theme-proxy plugin-id file-id theme-id))))}
 
     :getSetById
-    {:schema [:tuple ::sm/uuid]
+    {:enumerable false
+     :schema [:tuple ::sm/uuid]
      :fn (fn [set-id]
            (let [set (u/locate-token-set file-id set-id)]
              (when (some? set)

plugins/apps/poc-tokens-plugin/src/app/app.component.html

@@ -18,12 +18,7 @@
         <ul data-handler="themes-list">
           @for (theme of themes; track theme.id) {
             <li class="body-m panel-item theme-item">
-              <span title="{{ theme.activeSets }}">
-                @if (theme.group) {
-                  {{ theme.group }} /
-                }
-                {{ theme.name }}
-              </span>
+              <span>{{ theme.group }} / {{ theme.name }}</span>
               <button
                 type="button"
                 data-appearance="secondary"

plugins/apps/poc-tokens-plugin/src/app/app.component.ts

@@ -10,7 +10,6 @@ type TokenTheme = {
   group: string;
   description: string;
   active: boolean;
-  activeSets: string;
 };
 
 type TokenSet = {

plugins/apps/poc-tokens-plugin/src/plugin.ts

@@ -61,13 +61,11 @@ function loadLibrary() {
   const themes = tokensCatalog.themes;
 
   const themesData = themes.map((theme) => {
-    const activeSets = theme.activeSets.map((set) => set.name).join(', ');
     return {
       id: theme.id,
       group: theme.group,
       name: theme.name,
       active: theme.active,
-      activeSets: activeSets,
     };
   });