WIP arrow api

2026-09-08 11:54:36 -04:00 · 2026-07-16 12:54:03 +02:00 · 2026-07-16 12:44:48 +02:00 · 2026-07-16 12:38:32 +02:00 · 2026-07-16 08:56:03 +02:00 · 2026-07-16 07:27:55 +02:00
22 changed files with 2685 additions and 5 deletions
@@ -64,13 +64,19 @@
  ;; Pretty Print specs
  pretty-spec/pretty-spec {:mvn/version "0.1.4"}
  software.amazon.awssdk/s3 {:mvn/version "2.46.18"}
-  software.amazon.awssdk/sts {:mvn/version "2.46.18"}}
+  software.amazon.awssdk/sts {:mvn/version "2.46.18"}
+
+  com.ladybugdb/lbug {:mvn/version "0.18.0"}
+  ;; Required by Arrow RootAllocator (lbug only pulls arrow-memory-core).
+  org.apache.arrow/arrow-memory-netty {:mvn/version "18.2.0"}}

 :paths ["src" "resources" "target/classes"]
 :aliases
 {:dev
  {:jvm-opts ["--sun-misc-unsafe-memory-access=allow"
-              "--enable-native-access=ALL-UNNAMED"]
+              "--enable-native-access=ALL-UNNAMED"
+              ;; Arrow jars are on the classpath (unnamed module), not module-path.
+              "--add-opens=java.base/java.nio=ALL-UNNAMED"]
   :extra-deps
   {com.bhauman/rebel-readline {:mvn/version "0.1.11"}
    clojure-humanize/clojure-humanize {:mvn/version "0.2.2"}
@@ -0,0 +1,91 @@
+;; Spike: Ladybug createArrowTable → native COPY (no CSV).
+;; Run inside devenv from backend/:
+;;   clojure -M:dev -m graph-arrow-spike
+;;
+;; Requires --add-opens=java.base/java.nio=ALL-UNNAMED
+;; (configured in deps.edn :dev :jvm-opts).
+;;
+;; FINDING: Arrow tables created with createArrowTable are NOT available as direct
+;; identifiers in COPY statements, but ARE available as nodes in MATCH queries.
+;; SOLUTION: Use "COPY table FROM (MATCH (n:arrow_table) RETURN ...)" pattern.
+
+(ns graph-arrow-spike
+  (:gen-class)
+  (:import
+   (com.ladybugdb Connection Database QueryResult)
+   (java.nio.charset StandardCharsets)
+   (java.util ArrayList List)
+   (org.apache.arrow.memory RootAllocator)
+   (org.apache.arrow.vector VarCharVector VectorSchemaRoot)
+   (org.apache.arrow.vector.types.pojo ArrowType$Utf8 Field FieldType Schema)))
+
+(defn- check!
+  [^QueryResult result label]
+  (when-not (.isSuccess result)
+    (throw (ex-info (str label ": " (.getErrorMessage result))
+                    {:label label
+                     :err (.getErrorMessage result)})))
+  result)
+
+(defn- page-root
+  [^RootAllocator alloc]
+  (let [varchar-type (org.apache.arrow.vector.types.pojo.ArrowType$Utf8.)
+        field-id     (Field. "id" (FieldType/nullable varchar-type) nil)
+        field-name   (Field. "name" (FieldType/nullable varchar-type) nil)
+        schema       (Schema. [field-id field-name])
+        root         (VectorSchemaRoot/create schema alloc)
+        ^VarCharVector idv (.getVector root "id")
+        ^VarCharVector nv  (.getVector root "name")]
+    (.allocateNew idv 2)
+    (.allocateNew nv 2)
+    (.setSafe idv 0 (.getBytes "p1" StandardCharsets/UTF_8))
+    (.setSafe idv 1 (.getBytes "p2" StandardCharsets/UTF_8))
+    (.setSafe nv 0 (.getBytes "Home" StandardCharsets/UTF_8))
+    (.setSafe nv 1 (.getBytes "About" StandardCharsets/UTF_8))
+    (.setValueCount idv 2)
+    (.setValueCount nv 2)
+    (.setRowCount root 2)
+    root))
+
+(defn- try-query!
+  [^Connection conn cypher label]
+  (with-open [^QueryResult r (.query conn cypher)]
+    (println label
+             "success?" (.isSuccess r)
+             "tuples" (when (.isSuccess r) (.getNumTuples r))
+             "err" (when-not (.isSuccess r) (.getErrorMessage r)))
+    (.isSuccess r)))
+
+(defn -main
+  [& _]
+  (with-open [^RootAllocator alloc (RootAllocator.)
+              ^Database db (Database.)
+              ^Connection conn (Connection. db)]
+    (.setQueryTimeout conn 0)
+    (println "=== 1) native DDL ===")
+    (with-open [r (.query conn "CREATE NODE TABLE Page(id STRING, name STRING, PRIMARY KEY(id));")]
+      (check! r "ddl"))
+
+    (println "=== 2) createArrowTable staging ===")
+    (let [root2 (page-root alloc)
+          batches (doto (ArrayList.) (.add root2))]
+      (with-open [r (.createArrowTable conn "stg_Page" ^List batches alloc)]
+        (check! r "createArrowTable"))
+
+      (println "=== 3) query staging ===")
+      (try-query! conn "MATCH (n:stg_Page) RETURN n.id, n.name;" "stg")
+
+      (println "=== 4) COPY Page FROM Arrow table via MATCH subquery ===")
+      (try-query! conn
+                  "COPY Page FROM (MATCH (n:stg_Page) RETURN n.id AS id, n.name AS name);"
+                  "copy-via-match")
+
+      (println "=== 5) query native Page ===")
+      (try-query! conn "MATCH (n:Page) RETURN n.id, n.name;" "page")
+
+      (println "=== 6) dropArrowTable ===")
+      (with-open [r (.dropArrowTable conn "stg_Page")]
+        (println "drop success?" (.isSuccess r) "err" (.getErrorMessage r)))
+
+      (.close root2))
+    (println "DONE")))
@@ -222,6 +222,21 @@ Debug Main Page
        </div>
      </form>
    </fieldset>
+    <fieldset>
+      <legend>Export graph (Ladybug):</legend>
+      <desc>Given a FILE-ID, builds the graph projection and downloads
+      the `.lbug` database file.</desc>
+
+      <form method="get" action="/dbg/actions/graph-export">
+        <div class="row">
+          <input type="text" style="width:300px" name="file-id" placeholder="file-id" />
+        </div>
+        <div class="row">
+          <input type="submit" value="Download .lbug" />
+          <a href="/dbg/graph">Open graph console</a>
+        </div>
+      </form>
+    </fieldset>
    <fieldset>
      <legend>Import binfile:</legend>
      <desc>Import penpot file in binary format.</desc>
@@ -0,0 +1,390 @@
+{% extends "app/templates/base.tmpl" %}
+
+{% block title %}
+Graph Console
+{% endblock %}
+
+{% block content %}
+<nav>
+  <div class="title">
+    <h1>GRAPH CONSOLE (VERSION: {{version}})</h1>
+  </div>
+</nav>
+<main class="dashboard">
+  <section class="widget">
+    <p><a href="/dbg">&larr; Back to debug</a></p>
+
+    <fieldset>
+      <legend>Load graph in memory</legend>
+      <desc>
+        Projects the Penpot file into an in-memory Ladybug database for this
+        admin session. Loading a new file replaces the previous one.
+      </desc>
+      <form method="post" action="/dbg/actions/graph-load">
+        <div class="row">
+          <input type="text" style="width:420px" name="file-id"
+                 placeholder="file-id"
+                 value="{% if session %}{{session.file-id}}{% endif %}" />
+        </div>
+        <div class="row">                 
+          <input type="submit" value="Load" />
+        </div>
+      </form>
+      {% if session %}
+      <form method="post" action="/dbg/actions/graph-unload">
+        <div class="row">
+          <input type="submit" value="Unload" />
+        </div>
+      </form>
+      {% endif %}
+    </fieldset>
+
+    {% if session %}
+    <fieldset>
+      <legend>Loaded session</legend>
+      <desc>
+        <p>
+          File: <b>{{session.name}}</b> ({{session.file-id}})<br />
+          Loaded at revision: <b>{{session.revn}}</b><br />
+          Graph revision: <b id="graph-sync-revn">{% if session.graph-revn %}{{session.graph-revn}}{% else %}{{session.revn}}{% endif %}</b><br />
+          Schema: <b>{{session.schema-version}}</b><br />
+          Loaded at: <b>{{session.loaded-at}}</b>
+        </p>
+        <p id="graph-sync-status">
+          Feed: <b id="graph-ws-status">connecting…</b>
+          <span id="graph-sync-error" style="display:none; margin-left: 1em; color: #b91c1c;"></span>
+        </p>
+        <form id="graph-reload-form" method="post" action="/dbg/actions/graph-reload">
+          <input type="submit" value="Full reload (fallback)" />
+        </form>
+        {% if session.projection.stats %}
+        <p>
+          Projection:
+          documents={{session.projection.stats.documents}},
+          pages={{session.projection.stats.pages}},
+          shapes={{session.projection.stats.shapes}}
+        </p>
+        {% endif %}
+      </desc>
+    </fieldset>
+
+    <fieldset>
+      <legend>File changes (live)</legend>
+      <desc>
+        Subscribes to the workspace WebSocket feed for visibility. The backend
+        applies supported changes incrementally to the in-memory Ladybug graph
+        via msgbus (<code>:file-change</code>).
+      </desc>
+      <div id="graph-changelog-empty" style="color: #666;">Waiting for changes…</div>
+      <table id="graph-changelog" border="1" cellpadding="4" cellspacing="0"
+             style="border-collapse: collapse; width: 100%; display: none;">
+        <thead>
+          <tr>
+            <th>revn</th>
+            <th>changes</th>
+          </tr>
+        </thead>
+        <tbody id="graph-changelog-body"></tbody>
+      </table>
+    </fieldset>
+
+    <fieldset>
+      <legend>Cypher query</legend>
+      <form id="graph-query-form" method="post" action="/dbg/actions/graph-query">
+        <div class="row">
+          <textarea name="query" rows="8" style="width:100%; font-family: monospace;">{{query}}</textarea>
+        </div>
+        <div class="row">
+          <input type="submit" value="Run query" />
+        </div>
+      </form>
+    </fieldset>
+
+    <div id="graph-query-output">
+      {% if error %}
+      <fieldset>
+        <legend>Error</legend>
+        <pre>{{error}}</pre>
+      </fieldset>
+      {% endif %}
+
+      {% if query-result %}
+      <fieldset>
+        <legend>Results ({{query-result.row-count}} rows{% if query-result.truncated? %}, truncated{% endif %})</legend>
+        <table border="1" cellpadding="4" cellspacing="0" style="border-collapse: collapse; width: 100%;">
+          <thead>
+            <tr>
+              {% for column in query-result.columns %}
+              <th>{{column}}</th>
+              {% endfor %}
+            </tr>
+          </thead>
+          <tbody>
+            {% for row in query-result.rows %}
+            <tr>
+              {% for cell in row %}
+              <td><code>{{cell}}</code></td>
+              {% endfor %}
+            </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </fieldset>
+      {% endif %}
+    </div>
+    {% endif %}
+  </section>
+</main>
+
+{% if session %}
+<script>
+(function () {
+  const fileId = "{{session.file-id}}";
+  const sessionId = crypto.randomUUID();
+  const wsScheme = location.protocol === "https:" ? "wss:" : "ws:";
+  const wsUrl = wsScheme + "//" + location.host
+    + "/ws/notifications?session-id=" + sessionId;
+
+  const wsStatus = document.getElementById("graph-ws-status");
+  const syncRevnEl = document.getElementById("graph-sync-revn");
+  const syncErrorEl = document.getElementById("graph-sync-error");
+  const changelog = document.getElementById("graph-changelog");
+  const changelogBody = document.getElementById("graph-changelog-body");
+  const changelogEmpty = document.getElementById("graph-changelog-empty");
+
+  let ws = null;
+
+  function encodeTransitUuid(uuid) {
+    return "~u" + uuid;
+  }
+
+  function encodeSubscribe(fileId) {
+    return JSON.stringify({
+      "~:type": "~:subscribe-file",
+      "~:file-id": encodeTransitUuid(fileId)
+    });
+  }
+
+  function encodeUnsubscribe(fileId) {
+    return JSON.stringify({
+      "~:type": "~:unsubscribe-file",
+      "~:file-id": encodeTransitUuid(fileId)
+    });
+  }
+
+  function parseTransitValue(value) {
+    if (typeof value === "string") {
+      if (value.startsWith("~:")) return value.slice(2);
+      if (value.startsWith("~u")) return value.slice(2);
+    }
+    if (Array.isArray(value)) return value.map(parseTransitValue);
+    if (value && typeof value === "object") return parseTransitMap(value);
+    return value;
+  }
+
+  function parseTransitMap(obj) {
+    const out = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const name = key.startsWith("~:") ? key.slice(2) : key;
+      out[name] = parseTransitValue(value);
+    }
+    return out;
+  }
+
+  function summarizeChange(change) {
+    const parts = [change.type];
+    if (change.id) parts.push("id=" + change.id);
+    if (change.obj && change.obj.type) parts.push("shape=" + change.obj.type);
+    if (change.operations && change.operations.length) {
+      const attrs = change.operations
+        .map(function (op) { return op.attr; })
+        .filter(Boolean);
+      if (attrs.length) parts.push("attrs=" + attrs.join(","));
+    }
+    return parts.join(" ");
+  }
+
+  function summarizeChanges(changes) {
+    if (!changes || !changes.length) return "(empty)";
+    return changes.map(summarizeChange).join("; ");
+  }
+
+  function summarizeSkipped(skipped) {
+    if (!skipped) return "";
+    const items = Array.isArray(skipped) ? skipped : [skipped];
+    return items.map(function (item) {
+      if (!item || typeof item !== "object") return String(item);
+      const type = item.type || "unknown";
+      const reason = item.reason ? " (" + item.reason + ")" : "";
+      return String(type) + reason;
+    }).join("; ");
+  }
+
+  function refreshSyncStatus() {
+    fetch("/dbg/actions/graph-sync-status")
+      .then(function (resp) { return resp.text(); })
+      .then(function (text) {
+        const status = parseTransitMap(JSON.parse(text));
+        if (status["graph-revn"] !== undefined) {
+          syncRevnEl.textContent = String(status["graph-revn"]);
+        }
+        if (status.sync && status.sync.error) {
+          syncErrorEl.style.display = "inline";
+          syncErrorEl.textContent = "sync error: " + status.sync.error;
+        } else if (status.sync && status.sync["last-skipped"]
+                   && summarizeSkipped(status.sync["last-skipped"])) {
+          syncErrorEl.style.display = "inline";
+          syncErrorEl.textContent =
+            "some changes skipped: "
+            + summarizeSkipped(status.sync["last-skipped"])
+            + " (use full reload if needed)";
+        } else {
+          syncErrorEl.style.display = "none";
+          syncErrorEl.textContent = "";
+        }
+      })
+      .catch(function () {});
+  }
+
+  function appendChange(revn, summary) {
+    changelogEmpty.style.display = "none";
+    changelog.style.display = "table";
+
+    const row = document.createElement("tr");
+    const revnCell = document.createElement("td");
+    const changesCell = document.createElement("td");
+
+    revnCell.textContent = String(revn);
+    changesCell.textContent = summary;
+    row.appendChild(revnCell);
+    row.appendChild(changesCell);
+    changelogBody.appendChild(row);
+    row.scrollIntoView({ block: "nearest" });
+  }
+
+  function handleMessage(raw) {
+    let msg;
+    try {
+      msg = parseTransitMap(JSON.parse(raw));
+    } catch (_err) {
+      return;
+    }
+
+    if (msg.type !== "file-change" || msg["file-id"] !== fileId) return;
+
+    appendChange(msg.revn, summarizeChanges(msg.changes));
+    setTimeout(refreshSyncStatus, 150);
+  }
+
+  function subscribe() {
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(encodeSubscribe(fileId));
+    }
+  }
+
+  function connect() {
+    ws = new WebSocket(wsUrl);
+    wsStatus.textContent = "connecting…";
+
+    ws.addEventListener("open", function () {
+      wsStatus.textContent = "subscribed";
+      subscribe();
+    });
+
+    ws.addEventListener("message", function (event) {
+      handleMessage(event.data);
+    });
+
+    ws.addEventListener("close", function () {
+      wsStatus.textContent = "disconnected";
+    });
+
+    ws.addEventListener("error", function () {
+      wsStatus.textContent = "error";
+    });
+  }
+
+  function escapeHtml(text) {
+    return String(text)
+      .replace(/&/g, "&amp;")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/"/g, "&quot;");
+  }
+
+  function renderQueryOutput(data) {
+    const output = document.getElementById("graph-query-output");
+    if (!output) return;
+
+    if (data.error) {
+      output.innerHTML =
+        "<fieldset><legend>Error</legend>"
+        + "<pre>" + escapeHtml(data.error) + "</pre></fieldset>";
+      return;
+    }
+
+    const result = data["query-result"];
+    if (!result) {
+      output.innerHTML = "";
+      return;
+    }
+
+    const truncated = result["truncated?"] ? ", truncated" : "";
+    let html =
+      "<fieldset><legend>Results ("
+      + escapeHtml(String(result["row-count"]))
+      + " rows" + truncated + ")</legend>"
+      + "<table border=\"1\" cellpadding=\"4\" cellspacing=\"0\""
+      + " style=\"border-collapse: collapse; width: 100%;\">"
+      + "<thead><tr>";
+
+    (result.columns || []).forEach(function (column) {
+      html += "<th>" + escapeHtml(column) + "</th>";
+    });
+    html += "</tr></thead><tbody>";
+
+    (result.rows || []).forEach(function (row) {
+      html += "<tr>";
+      row.forEach(function (cell) {
+        html += "<td><code>" + escapeHtml(cell) + "</code></td>";
+      });
+      html += "</tr>";
+    });
+
+    html += "</tbody></table></fieldset>";
+    output.innerHTML = html;
+  }
+
+  const queryForm = document.getElementById("graph-query-form");
+  if (queryForm) {
+    queryForm.addEventListener("submit", function (event) {
+      event.preventDefault();
+      const formData = new FormData(queryForm);
+      fetch("/dbg/actions/graph-query", {
+        method: "POST",
+        headers: { "Accept": "application/json" },
+        body: formData
+      })
+        .then(function (resp) { return resp.text(); })
+        .then(function (text) {
+          renderQueryOutput(parseTransitMap(JSON.parse(text)));
+        })
+        .catch(function (err) {
+          renderQueryOutput({ error: String(err) });
+        });
+    });
+  }
+
+  connect();
+  refreshSyncStatus();
+
+  window.addEventListener("beforeunload", function () {
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(encodeUnsubscribe(fileId));
+      ws.close();
+    }
+  });
+})();
+</script>
+{% endif %}
+{% endblock %}
@@ -84,7 +84,8 @@ export JAVA_OPTS="\
       -XX:-OmitStackTraceInFastThrow  \
       --sun-misc-unsafe-memory-access=allow \
       --enable-preview \
-       --enable-native-access=ALL-UNNAMED";
+       --enable-native-access=ALL-UNNAMED \
+       --add-opens=java.base/java.nio=ALL-UNNAMED";

 function setup_minio() {
    if [ "${PENPOT_OBJECTS_STORAGE_BACKEND}" != "s3" ]; then
@@ -18,7 +18,7 @@ if [ -f ./environ ]; then
    source ./environ
 fi

-export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"
+export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS"

 ENTRYPOINT=${1:-app.main};

@@ -0,0 +1,223 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.bulk
+  "Bulk Ladybug ingest via COPY FROM CSV.
+
+  Node and relationship rows are written to a temporary staging directory
+  and loaded with one COPY statement per table (or per rel FROM/TO pair)."
+  (:require
+   [app.common.json :as json]
+   [app.graph.ladybug :as ladybug]
+   [app.graph.schema.nodes :as nodes]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [datoteka.fs :as fs])
+  (:import
+   java.io.File))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private copy-csv-options
+  "Ladybug COPY CSV options. QUOTE must be set explicitly or commas in
+  string fields are treated as column separators."
+  "HEADER=true, DELIM=',', QUOTE='\"'")
+
+(defn- csv-normalize-string
+  "Graph node names are single-line labels; flatten Penpot text newlines."
+  [s]
+  (-> (str s)
+      (str/replace #"\r\n" " ")
+      (str/replace #"\r" " ")
+      (str/replace #"\n" " ")))
+
+(defn- csv-escape-string
+  [s]
+  (str "\"" (str/replace (csv-normalize-string s) "\"" "\"\"") "\""))
+
+(defn- number-string
+  [v]
+  (if (== v (long v)) (str (long v)) (str (double v))))
+
+(defn- list-base-type
+  [lbug-type]
+  (when (and lbug-type (str/ends-with? lbug-type "[]"))
+    (subs lbug-type 0 (- (count lbug-type) 2))))
+
+(defn- list-item-string
+  [base-type item]
+  (case base-type
+    "UUID"   (str item)
+    "STRING" (str "\"" (str/replace (csv-normalize-string (str item)) "\"" "\"\"") "\"")
+    (#{"DOUBLE" "INT64"} base-type)
+    (number-string item)
+
+    (if (string? item)
+      (str "\"" (csv-normalize-string item) "\"")
+      (str item))))
+
+(defn- list-cell
+  [lbug-type v]
+  (when (some? v)
+    (let [base-type (list-base-type lbug-type)]
+      (if (empty? v)
+        "[]"
+        (str "[" (str/join ", " (map #(list-item-string base-type %) v)) "]")))))
+
+(defn- csv-cell*
+  "Raw COPY cell value before CSV quoting."
+  [table col v]
+  (let [lbug-type (nodes/column-ladybug-type table col)]
+    (cond
+      (#{"DOUBLE" "INT64"} lbug-type)
+      (when (number? v) (number-string v))
+
+      (= lbug-type "BOOLEAN")
+      (when (boolean? v) (str v))
+
+      (= lbug-type "UUID")
+      (when (some? v) (str v))
+
+      (= lbug-type "TIMESTAMP")
+      (when (some? v) (str v))
+
+      (list-base-type lbug-type)
+      (list-cell lbug-type v)
+
+      (nil? v)
+      ""
+
+      (= lbug-type "JSON")
+      (json/encode v)
+
+      (string? v)
+      (csv-normalize-string v)
+
+      (keyword? v)
+      (name v)
+
+      (uuid? v)
+      (str v)
+
+      (number? v)
+      (number-string v)
+
+      (boolean? v)
+      (str v)
+
+      (map? v)
+      (json/encode v)
+
+      (coll? v)
+      (json/encode v)
+
+      :else
+      (str v))))
+
+(defn- csv-cell
+  [table col v]
+  (let [cell (csv-cell* table col v)]
+    (cond
+      (nil? cell) ""
+      (string? cell) (csv-escape-string cell)
+      :else (csv-escape-string (str cell)))))
+
+(defn- csv-scalar-cell
+  [v]
+  (cond
+    (nil? v)     ""
+    (uuid? v)    (csv-escape-string (str v))
+    (string? v)  (csv-escape-string v)
+    (number? v)  (csv-escape-string (number-string v))
+    (boolean? v) (csv-escape-string (str v))
+    :else        (csv-escape-string (str v))))
+
+(defn- cypher-file-path
+  [^File file]
+  (-> (.getAbsolutePath file)
+      (str/replace "\\" "\\\\")
+      (str/replace "'" "\\'")))
+
+(defn- write-node-csv!
+  [^File file table rows]
+  (let [columns (nodes/column-keys table)]
+    (with-open [w (io/writer file :encoding "UTF-8")]
+      (.write w (str (str/join "," (map name columns)) "\n"))
+      (doseq [row rows]
+        (.write w (str (str/join "," (map #(csv-cell table % (get row %)) columns))
+                       "\n"))))))
+
+(defn- write-edge-csv!
+  [^File file edges]
+  (with-open [w (io/writer file :encoding "UTF-8")]
+    (.write w "from,to,position\n")
+    (doseq [{:keys [from-id to-id position]} edges]
+      (.write w (str (csv-scalar-cell from-id) ","
+                     (csv-scalar-cell to-id) ","
+                     (csv-scalar-cell position) "\n")))))
+
+(defn- delete-tree!
+  [path]
+  (when (fs/exists? path)
+    (doseq [f (reverse (file-seq (io/file path)))]
+      (.delete ^File f))))
+
+(defn staging-dir
+  "Directory for temporary COPY CSV files."
+  [db-path file-id]
+  (if (= db-path ":memory:")
+    (str (fs/path (System/getProperty "java.io.tmpdir")
+                  "penpot-graph-bulk"
+                  (str file-id)))
+    (str (fs/path (str db-path ".bulk") (str file-id)))))
+
+(defn- copy-node-table!
+  [conn table ^File csv-file]
+  (let [statement (str "COPY `" table "` FROM '" (cypher-file-path csv-file)
+                       "' (" copy-csv-options ");")]
+    (try
+      (ladybug/exec-on-connection! conn [statement])
+      (catch clojure.lang.ExceptionInfo e
+        (throw (ex-info (str "COPY node table failed: " table)
+                        (merge (ex-data e)
+                               {:table table
+                                :csv-file (.getAbsolutePath csv-file)})
+                        e))))))
+
+(defn- copy-edge-group!
+  [conn from-table to-table ^File csv-file]
+  (let [statement (str "COPY `IsChildOf` FROM '" (cypher-file-path csv-file) "' "
+                       "(from='" from-table "', to='" to-table "', "
+                       copy-csv-options ");")]
+    (try
+      (ladybug/exec-on-connection! conn [statement])
+      (catch clojure.lang.ExceptionInfo e
+        (throw (ex-info (str "COPY edge group failed: " from-table " -> " to-table)
+                        (merge (ex-data e)
+                               {:from-table from-table
+                                :to-table to-table
+                                :csv-file (.getAbsolutePath csv-file)})
+                        e))))))
+
+(defn load-projection!
+  "Load projected nodes and edges into an open Ladybug connection."
+  [conn {:keys [nodes edges]} staging-path]
+  (fs/create-dir staging-path)
+  (try
+    (doseq [[table rows] (sort-by key nodes)
+            :when (seq rows)]
+      (let [csv-file (io/file staging-path (str table ".csv"))]
+        (write-node-csv! csv-file table rows)
+        (copy-node-table! conn table csv-file)))
+    (doseq [[[from-table to-table] group]
+            (sort-by identity (group-by (juxt :from-table :to-table) edges))
+            :when (seq group)]
+      (let [csv-file (io/file staging-path
+                              (str "IsChildOf_" from-table "_" to-table ".csv"))]
+        (write-edge-csv! csv-file group)
+        (copy-edge-group! conn from-table to-table csv-file)))
+    (finally
+      (delete-tree! staging-path))))
@@ -0,0 +1,218 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.debug
+  "In-memory Ladybug sessions for the debug graph console."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.time :as ct]
+   [app.graph.ingest :as graph.ingest]
+   [app.graph.ladybug :as ladybug]
+   [app.graph.sync :as graph.sync]
+   [app.msgbus :as mbus]
+   [clojure.string :as str]
+   [promesa.exec.csp :as sp])
+  (:import
+   com.ladybugdb.Connection
+   com.ladybugdb.Database
+   org.apache.arrow.memory.RootAllocator))
+
+(set! *warn-on-reflection* true)
+
+(def default-query
+  "MATCH (n:Document) RETURN n.id AS id, n.name AS name;")
+
+(defonce ^:private sessions
+  (atom {}))
+
+(defn- session-key
+  [profile-id]
+  (str profile-id))
+
+(defn- close-arrow-alloc!
+  [arrow-alloc]
+  (when arrow-alloc
+    (let [^RootAllocator alloc arrow-alloc
+          outstanding (.getAllocatedMemory alloc)]
+      (try
+        (.close alloc)
+        (catch Exception e
+          (l/wrn :hint "arrow allocator close failed on session destroy"
+                 :allocated-bytes outstanding
+                 :cause e))))))
+
+(defn- destroy-session!
+  "Release session resources.
+
+  Order matters for Arrow: close Ladybug Connection/Database first so any
+  retained Arrow staging buffers are released, then close the RootAllocator."
+  [{:keys [conn db sync-ch msgbus arrow-alloc]}]
+  (when sync-ch
+    (sp/close! sync-ch)
+    (when msgbus
+      (mbus/purge! msgbus [sync-ch])))
+  (when conn
+    (ex/ignoring (.close ^Connection conn)))
+  (when db
+    (ex/ignoring (.close ^Database db)))
+  (close-arrow-alloc! arrow-alloc))
+
+(defn- format-cell
+  [value]
+  (cond
+    (nil? value) "NULL"
+    (string? value) value
+    :else (str value)))
+
+(defn- format-query-result
+  [{:keys [columns rows truncated?]}]
+  {:columns (mapv str columns)
+   :rows    (mapv (fn [row]
+                    (mapv format-cell row))
+                  rows)
+   :truncated? truncated?
+   :row-count (count rows)})
+
+(defn- apply-file-change!
+  [conn profile-id {:keys [changes revn file-id]}]
+  (try
+    (some-> (get @sessions (session-key profile-id))
+            (as-> current
+                  (when (= file-id (:file-id current))
+                    (let [result (graph.sync/apply-changes!
+                                  conn (:index current) changes revn)
+                          sync-at (ct/now)]
+                      (swap! sessions assoc-in [(session-key profile-id) :index]
+                             (:index result))
+                      (swap! sessions update-in [(session-key profile-id) :meta]
+                             (fn [meta]
+                               (cond-> (-> meta
+                                           (update :sync dissoc :error)
+                                           (assoc-in [:sync :last-at] sync-at)
+                                           (assoc-in [:sync :last-applied] (:applied result))
+                                           (assoc-in [:sync :last-skipped] (:skipped result)))
+                                 (seq (:applied result))
+                                 (assoc :revn (:revn result)))))
+                      (when (seq (:skipped result))
+                        (l/dbg :hint "graph sync skipped changes"
+                               :file-id (str file-id)
+                               :revn revn
+                               :skipped (:skipped result)))))))
+    (catch Throwable cause
+      (l/wrn :hint "graph sync failed"
+             :file-id (str file-id)
+             :cause cause)
+      (swap! sessions assoc-in [(session-key profile-id) :meta :sync :error]
+             (ex-message cause)))))
+
+(defn- start-sync-loop!
+  [{:keys [conn profile-id file-id] :as session}]
+  (if-let [msgbus (:msgbus session)]
+    (let [sync-ch (sp/chan :buf (sp/dropping-buffer 64))]
+      (mbus/sub! msgbus :topic file-id :chan sync-ch)
+      (sp/go-loop []
+        (when-let [message (sp/take! sync-ch)]
+          (when (= :file-change (:type message))
+            (apply-file-change! conn profile-id message)))
+        (recur))
+      (assoc session :sync-ch sync-ch))
+    session))
+
+(defn session-info
+  "Return a public view of the current session for `profile-id`, if any."
+  [profile-id]
+  (when-let [{:keys [file-id meta loaded-at index]} (get @sessions (session-key profile-id))]
+    {:file-id        file-id
+     :name           (:name meta)
+     :revn           (:revn meta)
+     :graph-revn     (:revn index)
+     :schema-version (:schema-version meta)
+     :projection     (:projection meta)
+     :sync           (:sync meta)
+     :loaded-at      (ct/format-inst loaded-at :iso)}))
+
+(defn sync-status
+  "Return incremental sync status for the active session."
+  [profile-id]
+  (when-let [session (get @sessions (session-key profile-id))]
+    (let [{:keys [file-id meta index loaded-at]} session]
+      {:file-id    file-id
+       :revn       (:revn meta)
+       :graph-revn (:revn index)
+       :sync       (:sync meta)
+       :loaded-at  (ct/format-inst loaded-at :iso)})))
+
+(defn unload-session!
+  "Close and discard the in-memory graph for `profile-id`."
+  [profile-id]
+  (when-let [session (get @sessions (session-key profile-id))]
+    (destroy-session! session))
+  (swap! sessions dissoc (session-key profile-id)))
+
+(defn load-session!
+  "Ingest `file-id` into a new in-memory Ladybug database for `profile-id`.
+
+  Uses Arrow by default. The Arrow RootAllocator is owned by the session and
+  closed only after the Ladybug Database (on unload/reload); closing it while
+  the connection is still open leaks direct memory on every load."
+  [cfg profile-id file-id]
+  (unload-session! profile-id)
+  (let [^Database db (Database.)
+        ^Connection conn (Connection. db)
+        ^RootAllocator arrow-alloc (RootAllocator.)
+        msgbus (::mbus/msgbus cfg)]
+    (.setQueryTimeout conn 0)
+    (ladybug/ensure-extensions! conn)
+    (try
+      (let [meta  (graph.ingest/ingest-on-connection! cfg conn file-id
+                                                      :db-path ":memory:"
+                                                      :skip-stats? true
+                                                      :skip-validation? true
+                                                      :use-arrow? true
+                                                      :arrow-alloc arrow-alloc)
+            index (graph.sync/build-index file-id (:revn meta) (:projection meta))
+            meta  (update meta :projection select-keys [:stats])
+            session
+            (-> {:db db
+                 :conn conn
+                 :arrow-alloc arrow-alloc
+                 :file-id file-id
+                 :meta meta
+                 :index index
+                 :msgbus msgbus
+                 :profile-id profile-id
+                 :loaded-at (ct/now)}
+                start-sync-loop!)]
+        (swap! sessions assoc (session-key profile-id) session)
+        meta)
+      (catch Throwable cause
+        (destroy-session! {:conn conn :db db :arrow-alloc arrow-alloc :msgbus msgbus})
+        (throw cause)))))
+
+(defn query-session!
+  "Run `statement` against the in-memory graph for `profile-id`."
+  [profile-id statement]
+  (when (str/blank? statement)
+    (ex/raise :type :validation
+              :code :missing-query
+              :hint "cypher query is required"))
+  (if-let [{:keys [conn]} (get @sessions (session-key profile-id))]
+    (-> (ladybug/query-on-connection! conn statement)
+        format-query-result)
+    (ex/raise :type :not-found
+              :code :graph-session-not-loaded
+              :hint "load a file graph before running queries")))
+
+(defn console-context
+  "Build template data for the graph debug console page."
+  [profile-id & {:keys [query query-result error message]}]
+  {:session       (session-info profile-id)
+   :query         (or query default-query)
+   :query-result  query-result
+   :error         error
+   :message       message
+   :default-query default-query})
@@ -0,0 +1,135 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.ingest
+  "Penpot file -> Ladybug graph projection."
+  (:require
+   [app.binfile.common :as bfc]
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.types.file :as ctf]
+   [app.db :as db]
+   [app.graph.arrow :as graph.arrow]
+   [app.graph.arrow-simple :as arrow-simple]
+   [app.graph.bulk :as bulk]
+   [app.graph.ladybug :as ladybug]
+   [app.graph.project.document :as project.document]
+   [app.graph.project.transforms :as project.transforms]
+   [app.graph.schema :as schema]
+   [app.graph.stats :as stats]
+   [app.srepl.helpers :as h])
+  (:import
+   com.ladybugdb.Connection
+   org.apache.arrow.memory.BufferAllocator))
+
+(defn- fetch-file!
+  [system file-id]
+  (let [file-id (h/parse-uuid file-id)
+        file    (db/run! system #(bfc/get-file % file-id :realize? true))]
+    (when-not file
+      (ex/raise :type :not-found
+                :code :file-not-found
+                :file-id (str file-id)))
+    (when-not (:data file)
+      (ex/raise :type :validation
+                :code :file-without-data
+                :hint "file has no data to project"
+                :file-id (str file-id)))
+    [file-id file]))
+
+(defn- ingest-on-connection*!
+  [system ^Connection conn file-id
+   {:keys [db-path skip-stats? skip-validation? use-arrow? arrow-alloc]
+    :or   {skip-stats? true use-arrow? true}}]
+  (let [[file-id file] (fetch-file! system file-id)
+        db-path        (or db-path (ladybug/db-path-for-file file-id))
+        data           (:data file)]
+    (when-not skip-validation?
+      (ctf/check-file-data data))
+    (l/inf :hint "graph ingest"
+           :file-id (str file-id)
+           :revn (:revn file)
+           :db-path db-path
+           :schema schema/schema-version
+           :use-arrow? use-arrow?)
+    (let [ddl (schema/ddl-statements)
+          {:keys [nodes edges stats]}
+          (project.document/projection-data data file)]
+      (ladybug/exec-on-connection! conn ddl)
+
+      (if use-arrow?
+        (do
+          (when-not arrow-alloc
+            (ex/raise :type :internal
+                      :code :arrow-allocator-unavailable
+                      :hint "Arrow ingest requires an Arrow allocator"))
+          (l/inf :hint "Using simple Arrow-based projection loading")
+          (arrow-simple/load-projection-with-arrow-simple!
+           conn {:nodes nodes :edges edges} ^BufferAllocator arrow-alloc))
+        (let [staging-path (bulk/staging-dir db-path file-id)]
+          (l/inf :hint "Using CSV-based projection loading")
+          (bulk/load-projection! conn {:nodes nodes :edges edges} staging-path)))
+
+      (ladybug/exec-on-connection! conn ["CHECKPOINT;"])
+      {:file-id        file-id
+       :revn           (:revn file)
+       :name           (or (:name data) (:name file))
+       :db-path        db-path
+       :schema-version schema/schema-version
+       :projection     {:stats stats
+                        :nodes nodes
+                        :edges edges}
+       :transforms     (project.transforms/apply-transforms! system db-path data file)
+       :stats          (when-not skip-stats?
+                         (stats/summarize-connection conn))})))
+
+(defn ingest-on-connection!
+  "Project `file-id` into an already open Ladybug `conn`.
+
+  When `:use-arrow?` is true and no `:arrow-alloc` is supplied, a temporary
+  RootAllocator is created for this call and closed afterwards."
+  [system ^Connection conn file-id & {:keys [use-arrow? arrow-alloc] :as opts
+                                      :or   {use-arrow? true}}]
+  (let [opts (cond-> opts
+               (nil? (:use-arrow? opts))
+               (assoc :use-arrow? true))]
+    (if (and use-arrow? (nil? arrow-alloc))
+      (graph.arrow/with-allocator!
+        (fn [alloc]
+          (ingest-on-connection*! system conn file-id
+                                  (assoc opts :arrow-alloc alloc))))
+      (ingest-on-connection*! system conn file-id opts))))
+
+(defn ingest-file!
+  "Ingest a file into a Ladybug database.
+
+  By default the returned `:projection` only keeps `:stats` (not the full
+  `:nodes`/`:edges` maps) so REPL/`*1*` does not retain huge projections.
+  Pass `:keep-projection? true` when callers need the raw projection.
+
+  Arrow allocators are closed after the Ladybug connection/database so
+  staging buffers are not retained for the process lifetime."
+  [system file-id & {:keys [db-path reset-db? skip-stats? skip-validation? use-arrow? keep-projection?]
+                     :or   {reset-db? true use-arrow? true}}]
+  (let [db-path (or db-path (ladybug/db-path-for-file (h/parse-uuid file-id)))
+        run     (fn [arrow-alloc]
+                  (ladybug/with-connection! db-path
+                    (fn [conn]
+                      (cond-> (ingest-on-connection*!
+                               system conn file-id
+                               {:db-path db-path
+                                :skip-stats? skip-stats?
+                                :skip-validation? skip-validation?
+                                :use-arrow? use-arrow?
+                                :arrow-alloc arrow-alloc})
+                        (not keep-projection?)
+                        (update :projection select-keys [:stats])))))]
+    (when reset-db?
+      (ladybug/reset-db-path! db-path))
+    (if use-arrow?
+      ;; Allocator outside connection: close after Ladybug drops Arrow tables.
+      (graph.arrow/with-allocator! run)
+      (run nil))))
@@ -0,0 +1,269 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.ladybug
+  "Ladybug access layer for graph-backed Penpot.
+
+  Uses the embedded Java API (`com.ladybugdb/lbug`)."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.json :as json]
+   [clojure.string :as str]
+   [datoteka.fs :as fs])
+  (:import
+   com.ladybugdb.Connection
+   com.ladybugdb.Database
+   com.ladybugdb.FlatTuple
+   com.ladybugdb.QueryResult
+   com.ladybugdb.Value))
+
+(set! *warn-on-reflection* true)
+
+(defn default-graph-dir
+  []
+  (or (System/getenv "PENPOT_GRAPH_DIR") "/tmp/penpot-graph"))
+
+(defn db-path-for-file
+  [file-id]
+  (str (fs/path (default-graph-dir) (str file-id ".lbug"))))
+
+(defn- memory-db-path?
+  [db-path]
+  (= db-path ":memory:"))
+
+(defn reset-db-path!
+  [db-path]
+  (when-not (memory-db-path? db-path)
+    (when (fs/exists? db-path)
+      (fs/delete db-path))))
+
+(defn escape-cypher-string
+  [s]
+  (-> (str s)
+      (str/replace "\\" "\\\\")
+      (str/replace "'" "\\'")))
+
+(defn format-uuid
+  [id]
+  (str "uuid('" (str id) "')"))
+
+(defn format-string
+  [s]
+  (str "'" (escape-cypher-string s) "'"))
+
+(defn format-int
+  [n]
+  (str (long n)))
+
+(defn format-number
+  [n]
+  (if (== n (long n))
+    (format-int n)
+    (str (double n))))
+
+(defn format-json
+  [v]
+  (str "json('" (escape-cypher-string (json/encode v)) "')"))
+
+(defn format-value
+  [v]
+  (cond
+    (nil? v)     "NULL"
+    (uuid? v)    (format-uuid v)
+    (string? v)  (format-string v)
+    (number? v)  (format-number v)
+    (boolean? v) (if v "true" "false")
+    (keyword? v) (format-string (name v))
+    (map? v)     (format-json v)
+    (coll? v)    (format-json v)
+    :else        (format-string (str v))))
+
+(defn format-typed-value
+  [ladybug-type v]
+  (cond
+    (= ladybug-type "JSON") (format-json v)
+    (and (string? ladybug-type)
+         (str/ends-with? ladybug-type "[]")) (format-json v)
+    :else (format-value v)))
+
+(defn- ensure-semicolon
+  [statement]
+  (let [s (str/trim (str statement))]
+    (if (str/ends-with? s ";") s (str s ";"))))
+
+(defn- value->clj
+  [^Value value]
+  (when-not (.isNull value)
+    (let [v (.getValue value)]
+      (cond
+        (instance? Long v)    v
+        (instance? Integer v) (long v)
+        (instance? Double v)  v
+        :else                 v))))
+
+(defn- check-success!
+  [^QueryResult result statement]
+  (when-not (.isSuccess result)
+    (let [err (.getErrorMessage result)]
+      (ex/raise :type :internal
+                :code :ladybug-query-failed
+                :hint (str "Ladybug query failed: " err)
+                :statement statement
+                :err err))))
+
+(defn- query-columns
+  [^QueryResult result]
+  (let [ncols (.getNumColumns result)]
+    (vec (for [i (range ncols)]
+           (.getColumnName result (long i))))))
+
+(defn- query-row
+  [^FlatTuple tuple ncols]
+  (vec (for [i (range ncols)]
+         (with-open [^Value value (.getValue tuple (long i))]
+           (value->clj value)))))
+
+(def ^:private default-query-max-rows 200)
+
+(defn- read-query-rows
+  [^QueryResult result ncols max-rows]
+  (loop [rows [] n 0]
+    (if (and (< n max-rows) (.hasNext result))
+      (let [row (with-open [^FlatTuple tuple (.getNext result)]
+                  (query-row tuple ncols))]
+        (recur (conj rows row) (inc n)))
+      rows)))
+
+(defn query-on-connection!
+  "Execute a Cypher query on `conn` and return tabular results.
+
+  Returns `{:columns [...] :rows [[...] ...] :truncated? bool}`."
+  [^Connection conn statement & {:keys [max-rows]
+                                 :or   {max-rows default-query-max-rows}}]
+  (let [cypher (ensure-semicolon statement)]
+    (with-open [^QueryResult result (.query conn cypher)]
+      (check-success! result cypher)
+      (let [ncols   (long (.getNumColumns result))
+            columns (query-columns result)
+            rows    (read-query-rows result ncols max-rows)
+            total   (long (.getNumTuples result))]
+        {:columns    columns
+         :rows       rows
+         :truncated? (and (pos? total) (> total (count rows)))}))))
+
+(def ^:private default-query-timeout-ms
+  "0 disables query timeout (recommended for bulk COPY ingest)."
+  0)
+
+(defn- scalar-value
+  [^Connection conn statement]
+  (let [cypher (ensure-semicolon statement)]
+    (with-open [^QueryResult result (.query conn cypher)]
+      (check-success! result cypher)
+      (when (.hasNext result)
+        (with-open [^FlatTuple tuple (.getNext result)]
+          (with-open [^Value value (.getValue tuple 0)]
+            (value->clj value)))))))
+
+(defn- extension-statement-ok?
+  [err-msg]
+  (let [err (str/lower-case (or err-msg ""))]
+    (or (str/includes? err "already loaded")
+        (str/includes? err "already installed"))))
+
+(defn- run-extension-statement!
+  [^Connection conn statement]
+  (let [cypher (ensure-semicolon statement)]
+    (with-open [^QueryResult result (.query conn cypher)]
+      (when-not (.isSuccess result)
+        (let [err (.getErrorMessage result)]
+          (when-not (extension-statement-ok? err)
+            (check-success! result cypher)))))))
+
+(defn ensure-extensions!
+  "Install and load Ladybug extensions required by graph ingest and sync."
+  [^Connection conn]
+  (run-extension-statement! conn "INSTALL json;")
+  (run-extension-statement! conn "LOAD json;"))
+
+(defn- run-statements!
+  [^Connection conn statements]
+  (doseq [statement statements]
+    (let [cypher (ensure-semicolon statement)]
+      (with-open [^QueryResult result (.query conn cypher)]
+        (check-success! result cypher)))))
+
+(defn- ensure-db-path!
+  [db-path]
+  (when-not (memory-db-path? db-path)
+    (fs/create-dir (fs/parent db-path))))
+
+(defn with-connection!
+  "Open a Ladybug connection for `db-path` and invoke `(f conn)`.
+
+  Options:
+  - `:query-timeout-ms` query timeout in milliseconds (default 0, disabled)
+
+  For `:memory:`, the database only lives for the duration of this call;
+  all reads and writes must happen inside `f`."
+  [db-path f & {:keys [query-timeout-ms]
+                :or   {query-timeout-ms default-query-timeout-ms}}]
+  (ensure-db-path! db-path)
+  (let [^Database db (if (memory-db-path? db-path)
+                       (Database.)
+                       (Database. (str db-path)))]
+    (try
+      (let [^Connection conn (Connection. db)]
+        (try
+          (.setQueryTimeout conn (long query-timeout-ms))
+          (ensure-extensions! conn)
+          (f conn)
+          (finally
+            (.close conn))))
+      (finally
+        (.close db)))))
+
+(defn exec-on-connection!
+  "Execute Cypher statements on an open Ladybug connection."
+  [^Connection conn statements]
+  (assert (sequential? statements) "statements should be a sequential collection")
+  (run-statements! conn statements))
+
+(defn query-scalar-on-connection!
+  "Execute a query expected to return a single scalar value on `conn`."
+  [^Connection conn statement]
+  (scalar-value conn statement))
+
+(defn exec!
+  "Execute Cypher statements against a Ladybug database.
+
+  `db-path` is either `:memory:` or a filesystem path to a `.lbug` database."
+  [db-path statements]
+  (with-connection! db-path
+    (fn [conn]
+      (exec-on-connection! conn statements))))
+
+(defn query-scalar!
+  "Execute a query expected to return a single scalar value."
+  [db-path statement]
+  (with-connection! db-path
+    (fn [conn]
+      (query-scalar-on-connection! conn statement))))
+
+(defn smoke-test!
+  "Run a minimal CREATE + count against Ladybug."
+  [& {:keys [db-path] :or {db-path ":memory:"}}]
+  (when-not (memory-db-path? db-path)
+    (reset-db-path! db-path))
+  (with-connection! db-path
+    (fn [^Connection conn]
+      (run-statements! conn
+                       ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));"
+                        "CREATE (:Person {name: 'Alice', age: 25});"
+                        "CREATE (:Person {name: 'Bob', age: 30});"])
+      {:db-path      db-path
+       :person-count (scalar-value conn
+                                   "MATCH (a:Person) RETURN count(a) AS c;")})))
@@ -0,0 +1,133 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.project.document
+  "Project a Penpot file-data map into Ladybug nodes and structural edges.
+
+  Projects Document, Page, the full shape tree (skipping the root frame),
+  and `IsChildOf` edges from shapes to their page or container parent."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.graph.schema.nodes :as nodes]))
+
+(def root-frame-id
+  uuid/zero)
+
+(defn- document-attrs
+  [file data]
+  (-> file
+      (assoc :id (or (:id data) (:id file)))
+      (dissoc :data)))
+
+(defn- page-attrs
+  [page index]
+  (-> page
+      (dissoc :objects)
+      (cond-> (some? index) (assoc :index (long index)))))
+
+(defn- shape-table
+  [shape]
+  (nodes/table-for-type (:type shape)))
+
+(defn- shape-node-attrs
+  [table shape]
+  (nodes/project-attrs table shape))
+
+(defn- container-table?
+  [table]
+  (contains? nodes/container-tables table))
+
+(defn- child-shape-ids
+  "Child ids in Penpot z-order (reversed from the stored :shapes list)."
+  [parent]
+  (when-let [shapes (:shapes parent)]
+    (vec (reverse shapes))))
+
+(defn- initial-acc
+  []
+  {:nodes {}
+   :edges []
+   :stats {:documents 0 :pages 0 :shapes 0}})
+
+(declare project-shape-ids)
+
+(defn- project-shape
+  [objects acc table shape parent-table parent-id position]
+  (let [shape-id (:id shape)
+        acc'     (-> acc
+                     (update-in [:nodes table] (fnil conj []) (shape-node-attrs table shape))
+                     (update :edges conj {:from-table table
+                                          :from-id    shape-id
+                                          :to-table   parent-table
+                                          :to-id      parent-id
+                                          :position   position})
+                     (update-in [:stats :shapes] inc))]
+    (if-let [child-ids (when (container-table? table)
+                         (child-shape-ids shape))]
+      (project-shape-ids objects acc' table shape-id child-ids)
+      acc')))
+
+(defn- project-shape-ids
+  [objects acc parent-table parent-id child-ids]
+  (reduce
+   (fn [acc [position shape-id]]
+     (if-let [shape (get objects shape-id)]
+       (if-let [table (shape-table shape)]
+         (project-shape objects acc table shape parent-table parent-id position)
+         (do
+           (l/wrn :hint "unsupported shape type for graph slice"
+                  :shape-id (str shape-id)
+                  :type (:type shape))
+           acc))
+       (do
+         (l/wrn :hint "missing shape in page objects"
+                :shape-id (str shape-id))
+         acc)))
+   acc
+   (map-indexed vector child-ids)))
+
+(defn- project-page
+  [acc doc-id page position]
+  (let [page-id     (:id page)
+        objects     (:objects page)
+        root        (get objects root-frame-id)
+        page-node   (nodes/project-attrs "Page" (page-attrs page position))
+        acc'        (-> acc
+                        (update-in [:nodes "Page"] (fnil conj []) page-node)
+                        (update :edges conj {:from-table "Page"
+                                             :from-id    page-id
+                                             :to-table   "Document"
+                                             :to-id      doc-id
+                                             :position   position})
+                        (update-in [:stats :pages] inc))]
+    (if-let [top-level-ids (child-shape-ids root)]
+      (project-shape-ids objects acc' "Page" page-id top-level-ids)
+      acc')))
+
+(defn projection-data
+  "Build node/edge rows for projecting `data` into Ladybug.
+
+  Returns `{:nodes {table [attrs ...]} :edges [...] :stats {...}}`."
+  [data file]
+  (let [doc-id  (or (:id data) (:id file))
+        doc-node (nodes/project-attrs "Document" (document-attrs file data))
+        pages   (seq (reverse (:pages data)))
+        acc0    (-> (initial-acc)
+                    (update-in [:nodes "Document"] (fnil conj []) doc-node)
+                    (assoc-in [:stats :documents] 1))
+        acc     (if (empty? pages)
+                  acc0
+                  (reduce (fn [acc [position page-id]]
+                            (if-let [page (get-in data [:pages-index page-id])]
+                              (project-page acc doc-id page position)
+                              (do
+                                (l/wrn :hint "missing page in pages-index"
+                                       :page-id (str page-id))
+                                acc)))
+                          acc0
+                          (map-indexed vector pages)))]
+    (select-keys acc [:nodes :edges :stats])))
@@ -0,0 +1,15 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.project.transforms
+  "Derived graph links (instances, tokens, nested containment, etc.).
+
+  Stub for now: beadpot's `apply_transformations` will be ported here.")
+
+(defn apply-transforms!
+  "Apply derived transformations to an already projected graph."
+  [_system _db-path _data _file]
+  {:transforms 0})
@@ -0,0 +1,60 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.report
+  (:require
+   [clojure.core :as c]))
+
+(defn- println!
+  [& lines]
+  (doseq [line lines]
+    (println line)))
+
+(defn- section-title
+  [title]
+  (println! (str "\n" title)
+            (str (apply str (repeat (count title) "─")))))
+
+(defn- kv-line
+  [k v]
+  (format "  %-14s %s" (str k ":") v))
+
+(defn- print-node-counts
+  [nodes]
+  (doseq [[table count] (sort-by first nodes)
+          :when (pos? (long count))]
+    (println! (kv-line table count))))
+
+(defn print-ingest!
+  "Pretty-print the result map returned by `app.graph.ingest/ingest-file!`."
+  [{:keys [file-id revn name db-path schema-version projection transforms stats]}]
+  (section-title "Graph ingest")
+  (println! (kv-line "File" (str name " (" file-id ")"))
+            (kv-line "Revision" revn)
+            (kv-line "Schema" schema-version)
+            (kv-line "Database" db-path))
+
+  (when-let [pstats (:stats projection)]
+    (section-title "Projection")
+    (doseq [[k v] (sort-by key pstats)]
+      (println! (kv-line (c/name k) v))))
+
+  (section-title "Transforms")
+  (println! (kv-line "Applied" (or (:transforms transforms) 0)))
+
+  (when stats
+    (section-title "Graph counts")
+    (when-let [nodes (:nodes stats)]
+      (println! "  Nodes")
+      (print-node-counts nodes))
+    (when-let [edges (:edges stats)]
+      (println! "  Edges")
+      (doseq [[rel count] (sort-by key edges)
+              :when (pos? (long count))]
+        (println! (kv-line (c/name rel) count)))))
+
+  (println!)
+  nil)
@@ -0,0 +1,30 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.schema
+  "Ladybug DDL facade for the graph-backed Penpot vertical slice.
+
+  Node metadata and DDL generation live in `app.graph.schema.nodes`."
+  (:require
+   [app.graph.schema.nodes :as nodes]))
+
+(def schema-version
+  nodes/schema-version)
+
+(def container-node-tables
+  nodes/container-tables)
+
+(def shape-node-tables
+  nodes/shape-tables)
+
+(def node-tables
+  (mapv (fn [{:keys [table schema]}]
+          {:name table :schema schema})
+        nodes/node-types))
+
+(defn ddl-statements
+  []
+  (nodes/ddl-statements))
@@ -0,0 +1,232 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.schema.nodes
+  "Single source of truth for graph node tables.
+
+  Each registry entry declares Penpot Malli sources plus projection
+  options (`:drop`, optional `:extra`). Derived artifacts — Ladybug
+  DDL, CSV columns, validation, type dispatch — all flow from that."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.schema :as sm]
+   [app.common.types.file :as ctf]
+   [app.common.types.page :as ctp]
+   [app.graph.schema.projection :as projection]
+   [app.graph.schema.types :as types]
+   [clojure.string :as str]))
+
+(def schema-version
+  "penpot-graph-slice-2")
+
+;; beadpot/graph/schemas.py drop_fields
+(def ^:private document-projection
+  {:source ctf/schema:file
+   :drop   [:data]})
+
+(def ^:private page-projection
+  {:source ctp/schema:page
+   :drop   [:objects]})
+
+(def ^:private shape-projection
+  {:drop [:type]})
+
+(def ^:private shape-node-types
+  [{:table "Frame"     :penpot-type :frame   :container? true}
+   {:table "Group"     :penpot-type :group   :container? true}
+   {:table "Boolean"   :penpot-type :bool    :container? true}
+   {:table "SVGRaw"    :penpot-type :svg-raw :container? true}
+   {:table "Rectangle" :penpot-type :rect}
+   {:table "Circle"    :penpot-type :circle}
+   {:table "Path"      :penpot-type :path}
+   {:table "Text"      :penpot-type :text}
+   {:table "Image"     :penpot-type :image}])
+
+(defn- resolve-schema
+  [{:keys [schema source drop extra penpot-type]}]
+  (or schema
+      (when penpot-type
+        (projection/project-shape-schema penpot-type
+                                         {:drop  drop
+                                          :extra extra}))
+      (projection/project-schema source
+                                 {:drop  drop
+                                  :extra extra})))
+
+(defn- shape-node-entry
+  [{:keys [table penpot-type container?] :as entry}]
+  (let [projection (-> shape-projection
+                       (merge (:projection entry))
+                       (assoc :penpot-type penpot-type))]
+    {:table       table
+     :pk          :id
+     :penpot-type penpot-type
+     :container?  container?
+     :projection  projection
+     :schema      (resolve-schema projection)}))
+
+(def node-types
+  "Ordered node registry."
+  (into [{:table "Document"
+          :pk    :id
+          :projection document-projection
+          :schema (resolve-schema document-projection)}
+         {:table "Page"
+          :pk    :id
+          :projection page-projection
+          :schema (resolve-schema page-projection)}]
+        (map shape-node-entry shape-node-types)))
+
+(def ^:private by-table
+  (into {} (map (juxt :table identity) node-types)))
+
+(def ^:private by-penpot-type
+  (into {} (keep (fn [{:keys [penpot-type table]}]
+                   (when penpot-type [penpot-type table]))
+                 node-types)))
+
+(def container-tables
+  (into #{} (comp (filter :container?) (map :table)) node-types))
+
+(def shape-tables
+  (into [] (comp (filter :penpot-type) (map :table)) node-types))
+
+(defn table-for-type
+  "Map a Penpot shape `:type` keyword to a Ladybug node table name."
+  [penpot-type]
+  (get by-penpot-type (keyword penpot-type)))
+
+(defn node-entry
+  [table]
+  (get by-table table))
+
+(defn projection-for
+  "Return the projection options map for `table`."
+  [table]
+  (:projection (node-entry table)))
+
+(defn- entry-child-schema
+  "Return the value schema from a Malli map entry (`[k s]` or `[k props s]`)."
+  [entry]
+  (if (> (count entry) 2)
+    (nth entry 2)
+    (nth entry 1)))
+
+(defn column-ladybug-type
+  "Ladybug column type for projected key `k` on `table`."
+  [table k]
+  (some (fn [entry]
+          (when (= k (first entry))
+            (types/ladybug-type (entry-child-schema entry))))
+        (projection/schema-map-entries (:schema (node-entry table)))))
+
+(defn column-keys
+  "Projected column keys for `table`, in registry order."
+  [table]
+  (mapv first (projection/schema-map-entries (:schema (node-entry table)))))
+
+(defn columns
+  "Projected column names for `table`, in registry order."
+  [table]
+  (mapv name (column-keys table)))
+
+(def ^:private validate-node-fn
+  (memoize
+   (fn [table]
+     (let [{:keys [schema]} (node-entry table)]
+       (sm/check-fn schema
+                    :type :validation
+                    :code (keyword "graph-node-projection" (str/lower-case table))
+                    :hint (str "invalid graph node projection for " table))))))
+
+(defn- projection-error-hint
+  [table explain]
+  (str "invalid graph node projection for " table
+       (when explain
+         (str "\n" (sm/humanize-explain explain)))))
+
+(defn validate-node
+  "Validate and return projected node attrs for `table`."
+  [table value]
+  (let [{:keys [schema]} (node-entry table)]
+    (try
+      ((validate-node-fn table) value)
+      (catch clojure.lang.ExceptionInfo e
+        (let [data    (ex-data e)
+              explain (or (::sm/explain data)
+                          (sm/explain schema value))]
+          (ex/raise :type :validation
+                    :code (keyword "graph-node-projection" (str/lower-case table))
+                    :hint (projection-error-hint table explain)
+                    :table table
+                    ::sm/explain explain
+                    :cause e))))))
+
+(defn- get-projected-attr
+  [attrs k]
+  (or (get attrs k)
+      (when (keyword? k) (get attrs (name k)))))
+
+(defn- raise-empty-projection!
+  [table attrs]
+  (ex/raise :type :validation
+            :code (keyword "graph-node-projection" (str/lower-case table))
+            :hint (str "empty graph node projection for " table
+                       "; columns=" (count (column-keys table))
+                       " shape-keys=" (vec (keys attrs)))))
+
+(defn project-attrs
+  "Select and validate the projected columns for `table` from `attrs`."
+  [table attrs]
+  (let [projected (into {}
+                        (keep (fn [k]
+                                (when-let [v (get-projected-attr attrs k)]
+                                  [k v]))
+                              (column-keys table)))]
+    (when (empty? projected)
+      (raise-empty-projection! table attrs))
+    (validate-node table projected)))
+
+(defn match-label
+  "Cypher node label for MATCH; backtick-wrapped when required by Ladybug."
+  [table]
+  (if (#{"Group" "Boolean"} table)
+    (str "`" table "`")
+    table))
+
+(defn cypher-property-key
+  "Backtick-wrapped property key for inline Cypher literals."
+  [k]
+  (str "`" (name k) "`"))
+
+(defn- create-node-table-ddl
+  [{:keys [table pk schema]}]
+  (let [cols (for [entry (projection/schema-map-entries schema)
+                   :let [k     (first entry)
+                         child (entry-child-schema entry)]]
+               (str "`" (name k) "` " (types/ladybug-type child)))]
+    (str "CREATE NODE TABLE `" table "` ("
+         (str/join ", " (concat cols [(str "PRIMARY KEY (`" (name pk) "`)")]))
+         ");")))
+
+(defn is-child-of-ddl
+  []
+  (str "CREATE REL TABLE `IsChildOf` ("
+       "FROM `Page` TO `Document`, "
+       (str/join ", "
+                 (concat
+                  (map (fn [shape]
+                         (str "FROM `" shape "` TO `Page`"))
+                       shape-tables)
+                  (for [shape shape-tables
+                        container container-tables]
+                    (str "FROM `" shape "` TO `" container "`"))))
+       ", `position` INT64);"))
+
+(defn ddl-statements
+  []
+  (conj (mapv create-node-table-ddl node-types)
+        (is-child-of-ddl)))
@@ -0,0 +1,85 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.schema.projection
+  "Derive Ladybug node column schemas from Penpot Malli sources.
+
+  Same model as beadpot's `drop_fields`: start from the canonical schema
+  and remove keys that must not become graph columns."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.schema :as sm]
+   [app.common.types.shape :as cts]
+   [malli.core :as m]))
+
+(def ^:private malli-opts sm/default-options)
+
+(defn- coerce-schema
+  "Normalize Malli sources to a compiled schema, unwrapping `:val` nodes."
+  [schema]
+  (loop [s (cond
+             (sm/schema? schema) schema
+             :else (sm/schema schema))]
+    (if (= :malli.core/val (sm/type s))
+      (recur (first (sm/children s)))
+      s)))
+
+(defn- unsupported-projection-schema!
+  [schema]
+  (ex/raise :type :internal
+            :code :unsupported-projection-schema
+            :hint (str "unsupported projection schema type: "
+                       (sm/type (coerce-schema schema)))))
+
+(defn schema-map-entries
+  "Map entries for `schema`, flattening `:merge` composites."
+  [schema]
+  (let [s (coerce-schema schema)]
+    (or (seq (sm/entries s))
+        (unsupported-projection-schema! schema))))
+
+(defn- select-projected-keys
+  "Project `schema` to a flat map schema, optionally dropping keys."
+  [schema drop-keys]
+  (let [s    (coerce-schema schema)
+        keys (if (seq drop-keys)
+               (remove (set drop-keys) (sm/keys s))
+               (sm/keys s))]
+    (sm/select-keys s (vec keys))))
+
+(defn shape-type-schema
+  "Return the compiled Penpot Malli branch for shape type `penpot-type`.
+
+  `m/entries` on the shape `:multi` yields MapEntries whose values are
+  compiled branch schemas (wrapped in `:val`). `m/children` returns raw
+  entry forms and must not be used here."
+  [penpot-type]
+  (let [kw    (keyword penpot-type)
+        multi (sm/schema cts/schema:shape-attrs)]
+    (or (some (fn [entry]
+                (when (= kw (key entry))
+                  (val entry)))
+              (m/entries multi malli-opts))
+        (ex/raise :type :validation
+                  :code :unknown-shape-type
+                  :hint (str "unknown penpot shape type: " kw)))))
+
+(defn project-schema
+  "Build a graph node schema from canonical Malli `source`.
+
+  Options:
+  - `:drop`  - keys removed from the source (beadpot `drop_fields`)
+  - `:extra` - optional extra `[:map ...]` merged on top"
+  [source {:keys [drop extra]}]
+  (let [projected (select-projected-keys source drop)]
+    (if extra
+      (sm/merge projected (coerce-schema extra))
+      projected)))
+
+(defn project-shape-schema
+  "Project `:drop` from the Penpot schema for `penpot-type`."
+  [penpot-type opts]
+  (project-schema (shape-type-schema penpot-type) opts))
@@ -0,0 +1,60 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.schema.types
+  "Map Malli schemas to Ladybug column types.
+
+  Analogue of beadpot's `get_ladybug_type` (util/ladybug.py)."
+  (:require
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.time :as ct]
+   [malli.core :as m]))
+
+(def ^:private malli-opts sm/default-options)
+
+(def ^:private base-type->ladybug
+  {::sm/uuid        "UUID"
+   ::sm/safe-number "DOUBLE"
+   ::sm/safe-double "DOUBLE"
+   ::sm/safe-int    "INT64"
+   ::sm/number      "DOUBLE"
+   ::sm/boolean     "BOOLEAN"
+   ::sm/int         "INT64"
+   ::ct/inst        "TIMESTAMP"
+   :uuid            "UUID"
+   :string          "STRING"
+   :int             "INT64"
+   :double          "DOUBLE"
+   :float           "DOUBLE"
+   :boolean         "BOOLEAN"
+   :keyword         "STRING"
+   :inst            "TIMESTAMP"})
+
+(defn- normalize-schema
+  [schema]
+  (let [s (sm/schema schema)]
+    (if (m/-ref-schema? s)
+      (recur (m/deref s malli-opts))
+      s)))
+
+(defn ladybug-type
+  "Return the Ladybug column type for a Malli child schema."
+  [schema]
+  (let [s (normalize-schema schema)
+        t (m/type s)]
+    (or (base-type->ladybug t)
+        (case t
+          (:maybe :and) (ladybug-type (first (m/children s malli-opts)))
+          (:vector :sequential :set)
+          (str (ladybug-type (first (m/children s malli-opts))) "[]")
+
+          :enum "STRING"
+          (:map :map-of) "JSON"
+          (do
+            (l/wrn :hint "unmapped malli type for ladybug column, defaulting to STRING"
+                   :malli-type t)
+            "STRING")))))
@@ -0,0 +1,33 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.stats
+  (:require
+   [app.graph.ladybug :as ladybug]
+   [app.graph.schema.nodes :as nodes]))
+
+(defn- count-on-connection
+  [conn statement]
+  (or (ladybug/query-scalar-on-connection! conn statement) 0))
+
+(defn summarize-connection
+  "Return node/edge counts using an open Ladybug connection."
+  [conn]
+  {:nodes (into {}
+                (map (fn [table]
+                       [table (count-on-connection
+                               conn
+                               (str "MATCH (n:" (nodes/match-label table) ") "
+                                    "RETURN count(n) AS " table "_c;"))])
+                     (map :table nodes/node-types)))
+   :edges {:IsChildOf (count-on-connection
+                       conn
+                       "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}})
+
+(defn summarize
+  "Return node/edge counts from the graph database."
+  [db-path]
+  (ladybug/with-connection! db-path summarize-connection))
@@ -0,0 +1,524 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.sync
+  "Incremental Ladybug graph updates from Penpot file-change events."
+  (:require
+   [app.common.logging :as l]
+   [app.common.uuid :as uuid]
+   [app.graph.ladybug :as ladybug]
+   [app.graph.schema.nodes :as nodes]
+   [clojure.string :as str])
+  (:import
+   com.ladybugdb.Connection))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private supported-change-types
+  #{:add-obj :mod-obj :del-obj :add-page :del-page :mod-page :mov-objects})
+
+(defn- shape-table
+  [shape]
+  (nodes/table-for-type (:type shape)))
+
+(defn- build-parent-map
+  [edges]
+  (into {}
+        (map (fn [{:keys [from-id to-id to-table]}]
+               [from-id {:parent-id to-id :parent-table to-table}]))
+        edges))
+
+(defn- build-children-map
+  [edges]
+  (reduce (fn [acc {:keys [from-id to-id]}]
+            (update acc to-id (fnil conj #{}) from-id))
+          {}
+          edges))
+
+(defn- resolve-page-id
+  [shape-id parents pages]
+  (loop [id shape-id]
+    (cond
+      (contains? pages id) id
+      (get parents id) (recur (:parent-id (parents id)))
+      :else nil)))
+
+(defn- node-attrs-id
+  [attrs]
+  (cond
+    (map? attrs) (or (:id attrs) (get attrs "id"))
+    (and (vector? attrs) (= 2 (count attrs)))
+    (let [[k v] attrs]
+      (when (or (= k :id) (= k "id")) v))))
+
+(defn- table-rows
+  "Normalize a projection table value to a vector of attribute maps."
+  [nodes table]
+  (let [rows (or (get nodes table) (get nodes (keyword table)))]
+    (cond
+      (nil? rows) []
+      (map? rows) [rows]
+      (sequential? rows) (vec rows)
+      :else [])))
+
+(defn- document-id-from-nodes
+  [nodes file-id]
+  (or (some node-attrs-id (table-rows nodes "Document"))
+      file-id))
+
+(defn- page-index-entry
+  [attrs]
+  (let [id (node-attrs-id attrs)]
+    [id {:id    id
+         :name  (:name attrs)
+         :index (long (:index attrs 0))}]))
+
+(defn- index-pages
+  [nodes]
+  (into {} (map page-index-entry (table-rows nodes "Page"))))
+
+(defn- shape-index-table?
+  [table]
+  (not (contains? #{"Document" "Page" :Document :Page} table)))
+
+(defn- shape-index-entry
+  [table attrs parents pages edges]
+  (let [shape-id (node-attrs-id attrs)
+        {:keys [parent-id parent-table]} (parents shape-id)
+        edge (first (filter #(= shape-id (:from-id %)) edges))]
+    [shape-id {:id            shape-id
+               :name          (:name attrs)
+               :table         table
+               :parent-id     parent-id
+               :parent-table  parent-table
+               :position      (long (:position edge 0))
+               :page-id       (resolve-page-id shape-id parents pages)}]))
+
+(defn- index-shapes
+  [nodes edges parents pages]
+  (reduce
+   (fn [acc [table _]]
+     (into acc (map #(shape-index-entry table % parents pages edges)
+                    (table-rows nodes table))))
+   {}
+   (filter (fn [[table _]] (shape-index-table? table)) nodes)))
+
+(defn build-index
+  "Build a sync index from a full graph projection."
+  [file-id revn {:keys [nodes edges]}]
+  (let [doc-id         (document-id-from-nodes nodes file-id)
+        pages          (index-pages nodes)
+        parents        (build-parent-map edges)
+        children-index (build-children-map edges)
+        shapes         (index-shapes nodes edges parents pages)]
+    {:file-id  file-id
+     :doc-id   doc-id
+     :revn     (long revn)
+     :pages    pages
+     :shapes   shapes
+     :children children-index}))
+
+
+(defn- format-node-value
+  [table k v]
+  (ladybug/format-typed-value (nodes/column-ladybug-type table k) v))
+
+(defn- create-node-statement
+  [table attrs]
+  (let [label (nodes/match-label table)
+        pairs (for [k (nodes/column-keys table)
+                    :let [v (get attrs k)]
+                    :when (some? v)]
+                (str (nodes/cypher-property-key k) ": "
+                     (format-node-value table k v)))]
+    (str "CREATE (:" label " {" (str/join ", " pairs) "});")))
+
+(defn- delete-node-statement
+  [table shape-id]
+  (str "MATCH (n:" (nodes/match-label table) " {id: " (ladybug/format-uuid shape-id) "}) "
+       "DETACH DELETE n;"))
+
+(defn- create-edge-statement
+  [{:keys [from-table from-id to-table to-id position]}]
+  (str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "}), "
+       "(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
+       "CREATE (s)-[:IsChildOf {position: " (ladybug/format-int position) "}]->(p);"))
+
+(defn- delete-edge-statement
+  [{:keys [from-table from-id to-table to-id]}]
+  (str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "})"
+       "-[r:IsChildOf]->"
+       "(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
+       "DELETE r;"))
+
+(defn- set-edge-position-statement
+  [{:keys [from-table from-id to-table to-id position]}]
+  (str "MATCH (s:" (nodes/match-label from-table) " {id: " (ladybug/format-uuid from-id) "})"
+       "-[r:IsChildOf]->"
+       "(p:" (nodes/match-label to-table) " {id: " (ladybug/format-uuid to-id) "}) "
+       "SET r.position = " (ladybug/format-int position) ";"))
+
+(defn- set-node-attr-statement
+  [table shape-id attr value]
+  (str "MATCH (s:" (nodes/match-label table) " {id: " (ladybug/format-uuid shape-id) "}) "
+       "SET s." (nodes/cypher-property-key attr) " = "
+       (format-node-value table attr value) ";"))
+
+(defn- set-page-name-statement
+  [page-id name]
+  (str "MATCH (p:Page {id: " (ladybug/format-uuid page-id) "}) "
+       "SET p.name = " (ladybug/format-string name) ";"))
+
+
+(defn- set-document-revision-statement
+  [doc-id revn]
+  (str "MATCH (d:Document {id: " (ladybug/format-uuid doc-id) "}) "
+       "SET d.revn = " (ladybug/format-int revn) ";"))
+
+(defn- resolve-parent-for-add
+  [index {:keys [parent-id frame-id page-id]}]
+  (let [pid (or parent-id frame-id)]
+    (if (or (nil? pid) (uuid/zero? pid))
+      (when page-id
+        {:parent-id page-id :parent-table "Page"})
+      (if-let [shape (get-in index [:shapes pid])]
+        {:parent-id pid :parent-table (:table shape)}
+        (when (get-in index [:pages pid])
+          {:parent-id pid :parent-table "Page"})))))
+
+(defn- default-position
+  [index parent-id]
+  (count (get-in index [:children parent-id] #{})))
+
+(defn- index-add-shape!
+  [index {:keys [id name table parent-id parent-table position page-id]}]
+  (-> index
+      (assoc-in [:shapes id]
+                {:id           id
+                 :name         name
+                 :table        table
+                 :parent-id    parent-id
+                 :parent-table parent-table
+                 :position     position
+                 :page-id      page-id})
+      (update :children update parent-id (fnil conj #{}) id)))
+
+(defn- index-remove-shape!
+  [index shape-id]
+  (if-let [shape (get-in index [:shapes shape-id])]
+    (-> index
+        (update :shapes dissoc shape-id)
+        (update :children update (:parent-id shape)
+                #(disj (or % #{}) shape-id))
+        (update :children dissoc shape-id))
+    index))
+
+(defn- index-add-page!
+  [index {:keys [id name doc-id] page-index :index}]
+  (-> index
+      (assoc-in [:pages id] {:id id :name name :index page-index})
+      (update :children update doc-id (fnil conj #{}) id)))
+
+(defn- index-move-shape!
+  [index shape-id {:keys [parent-id parent-table position page-id]}]
+  (let [old-parent (get-in index [:shapes shape-id :parent-id])]
+    (-> index
+        (assoc-in [:shapes shape-id :parent-id] parent-id)
+        (assoc-in [:shapes shape-id :parent-table] parent-table)
+        (assoc-in [:shapes shape-id :position] position)
+        (cond-> page-id (assoc-in [:shapes shape-id :page-id] page-id))
+        (update :children update old-parent #(disj (or % #{}) shape-id))
+        (update :children update parent-id (fnil conj #{}) shape-id))))
+
+(defn- mov-object-ids
+  [shapes]
+  (let [coll (cond
+               (nil? shapes) []
+               (sequential? shapes) shapes
+               (uuid? shapes) [shapes]
+               (map? shapes) (if-let [id (or (:id shapes) (get shapes "id"))]
+                               [id]
+                               [])
+               :else [])]
+    (into []
+          (keep (fn [shape]
+                  (when shape
+                    (if (uuid? shape) shape (:id shape)))))
+          coll)))
+
+(defn- mov-position
+  [idx parent-id {:keys [index after-shape]}]
+  (cond
+    (some? index) (long index)
+    after-shape (let [after-pos (get-in idx [:shapes after-shape :position])]
+                  (if (some? after-pos)
+                    (inc (long after-pos))
+                    (default-position idx parent-id)))
+    :else (default-position idx parent-id)))
+
+(defn- apply-mov-objects
+  [index {:keys [shapes page-id] :as change}]
+  (let [shape-ids (mov-object-ids shapes)
+        parent    (resolve-parent-for-add index
+                                          (assoc change
+                                                 :frame-id (:parent-id change)
+                                                 :page-id page-id))]
+    (cond
+      (empty? shape-ids)
+      {:index index :statements [] :applied? true}
+
+      (not parent)
+      {:index index :statements [] :applied? false :reason :missing-parent}
+
+      :else
+      (let [base-position (mov-position index (:parent-id parent) change)
+            parent-id     (:parent-id parent)
+            parent-table  (:parent-table parent)
+            page-id'      (or page-id
+                              (when (= parent-table "Page") parent-id)
+                              (get-in index [:shapes (first shape-ids) :page-id]))]
+        (loop [index       index
+               statements  []
+               shape-ids   (map-indexed vector shape-ids)]
+          (if-let [[offset shape-id] (first shape-ids)]
+            (if-let [shape (get-in index [:shapes shape-id])]
+              (let [position   (+ base-position (long offset))
+                    same-edge? (and (= parent-id (:parent-id shape))
+                                    (= parent-table (:parent-table shape))
+                                    (= position (:position shape)))
+                    edge       {:from-table   (:table shape)
+                                :from-id      shape-id
+                                :to-table     parent-table
+                                :to-id        parent-id
+                                :position     position}
+                    statements (if same-edge?
+                                 statements
+                                 (into statements
+                                       (if (= parent-id (:parent-id shape))
+                                         [(set-edge-position-statement edge)]
+                                         [(delete-edge-statement
+                                           {:from-table   (:table shape)
+                                            :from-id      shape-id
+                                            :to-table     (:parent-table shape)
+                                            :to-id        (:parent-id shape)})
+                                          (create-edge-statement edge)])))
+                    index      (if same-edge?
+                                 index
+                                 (index-move-shape! index shape-id
+                                                    {:parent-id    parent-id
+                                                     :parent-table parent-table
+                                                     :position     position
+                                                     :page-id      page-id'}))]
+                (recur index statements (rest shape-ids)))
+              (recur index statements (rest shape-ids)))
+            {:index      index
+             :statements statements
+             :applied?   true}))))))
+
+(defn- index-remove-page!
+  [index page-id]
+  (let [doc-id (:doc-id index)]
+    (-> index
+        (update :pages dissoc page-id)
+        (update :children update doc-id #(disj (or % #{}) page-id))
+        (update :children dissoc page-id))))
+
+(defn- mod-attrs-for-table
+  [table]
+  (disj (set (nodes/column-keys table)) :id))
+
+(defn- apply-add-obj
+  [index change]
+  (let [{:keys [id obj page-id] pos :index} change
+        table (shape-table obj)]
+    (if-not table
+      {:index index :statements [] :applied? false :reason :unsupported-shape-type}
+      (let [parent (resolve-parent-for-add index change)]
+        (if-not parent
+          {:index index :statements [] :applied? false :reason :missing-parent}
+          (let [position (long (or pos (default-position index (:parent-id parent))))
+                attrs    (nodes/project-attrs table (assoc obj :id id))
+                edge     (merge {:from-table table
+                                 :from-id    id
+                                 :to-table   (:parent-table parent)
+                                 :to-id      (:parent-id parent)
+                                 :position   position})]
+            {:index       (index-add-shape! index
+                                            {:id            id
+                                             :name          (:name attrs)
+                                             :table         table
+                                             :parent-id     (:parent-id parent)
+                                             :parent-table  (:parent-table parent)
+                                             :position      position
+                                             :page-id       (or page-id (when (= (:parent-table parent) "Page")
+                                                                          (:parent-id parent)))})
+             :statements  [(create-node-statement table attrs)
+                           (create-edge-statement edge)]
+             :applied?    true}))))))
+
+(defn- apply-mod-obj
+  [index {:keys [id operations]}]
+  (if-let [shape (get-in index [:shapes id])]
+    (let [table   (:table shape)
+          syncable (mod-attrs-for-table table)
+          set-ops (filter #(and (= :set (:type %))
+                                (contains? syncable (:attr %)))
+                          operations)]
+      (if (empty? set-ops)
+        {:index index :statements [] :applied? false :reason :unsupported-operations}
+        (let [updates    (into {} (map (juxt :attr :val) set-ops))
+              statements (for [[attr value] updates]
+                           (set-node-attr-statement table id attr value))
+              index'     (reduce (fn [idx [attr value]]
+                                   (assoc-in idx [:shapes id attr] value))
+                                 index
+                                 updates)]
+          {:index      index'
+           :statements statements
+           :applied?   true})))
+    {:index index :statements [] :applied? false :reason :missing-shape}))
+
+(defn- delete-order-deepest-first
+  [children root-id]
+  (letfn [(post-order [id]
+            (into (mapcat post-order (get children id #{}))
+                  [id]))]
+    (post-order root-id)))
+
+(defn- apply-del-obj
+  [index {:keys [id]}]
+  (if (get-in index [:shapes id])
+    (let [to-delete (delete-order-deepest-first (:children index) id)
+          statements
+          (vec (concat
+                (mapcat (fn [shape-id]
+                          (let [{:keys [table parent-id parent-table]}
+                                (get-in index [:shapes shape-id])]
+                            [(delete-edge-statement
+                              {:from-table   table
+                               :from-id      shape-id
+                               :to-table     parent-table
+                               :to-id        parent-id})
+                             (delete-node-statement table shape-id)]))
+                        to-delete)))]
+      {:index      (reduce index-remove-shape! index to-delete)
+       :statements statements
+       :applied?   true})
+    ;; Penpot emits one :del-obj per selected shape; an earlier change in the
+    ;; same batch may have already removed this node (e.g. parent + child).
+    {:index index :statements [] :applied? true}))
+
+(defn- apply-add-page
+  [index {:keys [id name page]}]
+  (let [page-id  (or id (:id page))
+        page     (or page {:id page-id :name name})
+        page     (nodes/validate-node "Page" {:id page-id
+                                              :name (or (:name page) "Page")
+                                              :index (count (:pages index))})
+        doc-id   (:doc-id index)
+        position (count (:pages index))
+        edge     {:from-table "Page"
+                  :from-id    page-id
+                  :to-table   "Document"
+                  :to-id      doc-id
+                  :position   position}]
+    {:index      (index-add-page! index
+                                  {:id      page-id
+                                   :name    (:name page)
+                                   :index   (:index page)
+                                   :doc-id  doc-id})
+     :statements [(create-node-statement "Page" page)
+                  (create-edge-statement edge)]
+     :applied?   true}))
+
+(defn- apply-del-page
+  [index {:keys [id]}]
+  (if (get-in index [:pages id])
+    (let [shape-ids (into #{}
+                          (comp (filter #(= id (get-in index [:shapes % :page-id])))
+                                (filter #(= "Page" (get-in index [:shapes % :parent-table]))))
+                          (keys (:shapes index)))
+          del-shapes
+          (reduce (fn [acc shape-id]
+                    (let [result (apply-del-obj acc {:type :del-obj :id shape-id})]
+                      (if (:applied? result)
+                        (-> acc
+                            (assoc :index (:index result))
+                            (update :statements into (:statements result)))
+                        acc)))
+                  {:index index :statements []}
+                  shape-ids)
+          statements
+          (conj (:statements del-shapes)
+                (delete-edge-statement {:from-table "Page"
+                                        :from-id    id
+                                        :to-table   "Document"
+                                        :to-id      (:doc-id index)})
+                (delete-node-statement "Page" id))]
+      {:index      (-> (:index del-shapes) (index-remove-page! id))
+       :statements statements
+       :applied?   true})
+    {:index index :statements [] :applied? false :reason :missing-page}))
+
+(defn- apply-mod-page
+  [index {:keys [id name]}]
+  (if (and (string? name) (get-in index [:pages id]))
+    {:index      (assoc-in index [:pages id :name] name)
+     :statements [(set-page-name-statement id name)]
+     :applied?   true}
+    {:index index :statements [] :applied? false :reason :unsupported-page-change}))
+
+(defn- apply-change
+  [index change]
+  (case (:type change)
+    :add-obj  (apply-add-obj index change)
+    :mod-obj  (apply-mod-obj index change)
+    :del-obj  (apply-del-obj index change)
+    :add-page (apply-add-page index change)
+    :del-page (apply-del-page index change)
+    :mod-page (apply-mod-page index change)
+    :mov-objects (apply-mov-objects index change)
+    {:index index :statements [] :applied? false :reason :unsupported-type}))
+
+(defn apply-changes!
+  "Apply Penpot `changes` to an open Ladybug `conn` and return the updated index.
+
+  Returns `{:index ... :revn ... :applied [...] :skipped [...]}`."
+  [^Connection conn index changes revn]
+  (when (> (long revn) (:revn index))
+    (l/wrn :hint "graph sync revn gap"
+           :file-id (str (:file-id index))
+           :index-revn (:revn index)
+           :change-revn revn))
+  (loop [index   index
+         applied []
+         skipped []
+         stmts   []
+         changes (seq changes)]
+    (if-let [change (first changes)]
+      (let [{:keys [index statements applied? reason]}
+            (apply-change index change)]
+        (recur index
+               (cond-> applied applied? (conj (:type change)))
+               (cond-> skipped (not applied?) (conj {:type (:type change) :reason reason}))
+               (cond-> stmts applied? (into statements))
+               (rest changes)))
+      (let [final-stmts (cond-> stmts
+                          (and (seq applied) (:doc-id index))
+                          (conj (set-document-revision-statement (:doc-id index) revn)))
+            index'      (if (seq applied)
+                          (assoc index :revn (long revn))
+                          index)]
+        (when (seq final-stmts)
+          (ladybug/exec-on-connection! conn final-stmts))
+        {:index   index'
+         :revn    (if (seq applied) (long revn) (:revn index'))
+         :applied applied
+         :skipped skipped}))))
+
+(defn supported-change?
+  [change]
+  (contains? supported-change-types (:type change)))
@@ -21,6 +21,8 @@
   [app.config :as cf]
   [app.db :as db]
   [app.features.file-migrations :as feat.fmig]
+   [app.graph.debug :as graph.debug]
+   [app.graph.ingest :as graph.ingest]
   [app.http.session :as session]
   [app.rpc.commands.auth :as auth]
   [app.rpc.commands.files-create :refer [create-file]]
@@ -33,6 +35,7 @@
   [app.storage.tmp :as tmp]
   [app.util.template :as tmpl]
   [cuerdas.core :as str]
+   [datoteka.fs :as fs]
   [datoteka.io :as io]
   [emoji.core :as emj]
   [integrant.core :as ig]
@@ -326,6 +329,110 @@
                         "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}}))))


+(defn graph-export-handler
+  "Build (or rebuild) the Ladybug graph for a file and stream the `.lbug`
+  database. MVP: synchronous ingest on each request."
+  [cfg {:keys [params]}]
+  (let [file-id (some-> params :file-id parse-uuid)]
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments
+                :hint "missing file-id"))
+
+    (let [{:keys [db-path]} (graph.ingest/ingest-file! cfg file-id :skip-stats? true)]
+      (when-not (fs/exists? db-path)
+        (ex/raise :type :internal
+                  :code :graph-file-not-found
+                  :hint "graph database file missing after ingest"
+                  :file-id (str file-id)
+                  :db-path db-path))
+      {::yres/status  200
+       ::yres/body    (io/input-stream db-path)
+       ::yres/headers {"content-type" "application/octet-stream"
+                       "content-disposition" (str "attachment; filename=" file-id ".lbug")}})))
+
+(defn- graph-console-response
+  [data]
+  {::yres/status  200
+   ::yres/headers {"content-type" "text/html; charset=utf-8"
+                   "x-robots-tag" "noindex"}
+   ::yres/body    (-> (io/resource "app/templates/graph-console.tmpl")
+                      (tmpl/render (assoc data :version (:full cf/version))))})
+
+(defn graph-console-handler
+  [_cfg {:keys [::session/profile-id]}]
+  (graph-console-response (graph.debug/console-context profile-id)))
+
+(defn graph-load-handler
+  [cfg {:keys [params ::session/profile-id]}]
+  (let [file-id (some-> (:file-id params) parse-uuid)]
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments
+                :hint "missing file-id"))
+    (graph.debug/load-session! cfg profile-id file-id)
+    {::yres/status  302
+     ::yres/headers {"location" "/dbg/graph"}}))
+
+(defn graph-unload-handler
+  [_cfg {:keys [::session/profile-id]}]
+  (graph.debug/unload-session! profile-id)
+  {::yres/status  302
+   ::yres/headers {"location" "/dbg/graph"}})
+
+(defn graph-reload-handler
+  "Re-ingest the currently loaded file into the in-memory graph session."
+  [cfg {:keys [::session/profile-id]}]
+  (if-let [file-id (some-> (graph.debug/session-info profile-id) :file-id)]
+    (do
+      (graph.debug/load-session! cfg profile-id file-id)
+      {::yres/status  302
+       ::yres/headers {"location" "/dbg/graph"}})
+    (ex/raise :type :not-found
+              :code :graph-session-not-loaded
+              :hint "load a file graph before reloading")))
+
+(defn graph-sync-status-handler
+  [_cfg {:keys [::session/profile-id]}]
+  (if-let [status (graph.debug/sync-status profile-id)]
+    {::yres/status  200
+     ::yres/headers {"content-type" "application/json; charset=utf-8"}
+     ::yres/body    (t/encode-str status {:type :json-verbose})}
+    {::yres/status  404
+     ::yres/headers {"content-type" "application/json; charset=utf-8"}
+     ::yres/body    (t/encode-str {:error "no-session"} {:type :json-verbose})}))
+
+(defn- json-request?
+  [request]
+  (some-> request
+          (yreq/get-header "accept")
+          (str/includes? "application/json")))
+
+(defn graph-query-handler
+  [_cfg {:keys [params ::session/profile-id] :as request}]
+  (let [query (:query params)]
+    (try
+      (let [result (graph.debug/query-session! profile-id query)]
+        (if (json-request? request)
+          {::yres/status  200
+           ::yres/headers {"content-type" "application/json; charset=utf-8"}
+           ::yres/body    (t/encode-str {:query         query
+                                         :query-result result}
+                                        {:type :json-verbose})}
+          (graph-console-response (graph.debug/console-context profile-id
+                                                               :query query
+                                                               :query-result result))))
+      (catch Throwable e
+        (let [error (or (:hint (ex-data e)) (ex-message e))]
+          (if (json-request? request)
+            {::yres/status  200
+             ::yres/headers {"content-type" "application/json; charset=utf-8"}
+             ::yres/body    (t/encode-str {:query query :error error}
+                                          {:type :json-verbose})}
+            (graph-console-response (graph.debug/console-context profile-id
+                                                                 :query query
+                                                                 :error error))))))))
+
 (defn import-handler
  [{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}]
  (when-not (contains? params :file)
@@ -535,7 +642,12 @@
  (letfn [(handle-error [cause]
            (when-let [data (ex-data cause)]
              (when (= :validation (:type data))
-                (str "Error: " (or (:hint data) (ex-message cause)) "\n"))))]
+                (let [hint (or (:hint data) (ex-message cause))
+                      explain (ex/explain data)]
+                  (str "Error: " hint
+                       (when (and explain (not (str/includes? hint explain)))
+                         (str "\n" explain))
+                       "\n")))))]
    {:name ::errors
     :compile
     (fn [& _params]
@@ -563,6 +675,7 @@
    ["" {:handler (partial index-handler cfg)}]
    ["/health" {:handler (partial health-handler cfg)}]
    ["/changelog" {:handler (partial changelog-handler cfg)}]
+    ["/graph" {:handler (partial graph-console-handler cfg)}]
    ["/error/:id" {:handler (partial error-handler cfg)}]
    ["/error" {:handler (partial error-list-handler cfg)}]
    ["/actions" {:middleware [[errors]]}
@@ -573,6 +686,12 @@
     ["/handle-team-features"
      {:handler (partial handle-team-features cfg)}]
     ["/file-export" {:handler (partial export-handler cfg)}]
+     ["/graph-export" {:handler (partial graph-export-handler cfg)}]
+     ["/graph-load" {:handler (partial graph-load-handler cfg)}]
+     ["/graph-query" {:handler (partial graph-query-handler cfg)}]
+     ["/graph-unload" {:handler (partial graph-unload-handler cfg)}]
+     ["/graph-reload" {:handler (partial graph-reload-handler cfg)}]
+     ["/graph-sync-status" {:handler (partial graph-sync-status-handler cfg)}]
     ["/file-import" {:handler (partial import-handler cfg)}]
     ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])

@@ -284,6 +284,7 @@
   ::http.debug/routes
   {::db/pool         (ig/ref ::db/pool)
    ::session/manager (ig/ref ::session/manager)
+    ::mbus/msgbus     (ig/ref ::mbus/msgbus)
    ::sto/storage     (ig/ref ::sto/storage)
    ::setup/props     (ig/ref ::setup/props)}

@@ -25,6 +25,9 @@
   [app.db.sql :as-alias sql]
   [app.features.fdata :as fdata]
   [app.features.file-snapshots :as fsnap]
+   [app.graph.ingest :as graph.ingest]
+   [app.graph.ladybug :as graph.ladybug]
+   [app.graph.report :as graph.report]
   [app.http.session :as session]
   [app.loggers.audit :as audit]
   [app.main :as main]
@@ -398,6 +401,43 @@
                        (println (sm/humanize-explain explain))
                        (ex/print-throwable cause))))))))

+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; GRAPH / LADYBUG
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn graph-smoke-test!
+  "Execute a basic Ladybug smoke test (CREATE + count).
+
+  Uses the embedded Ladybug Java API. Use :db-path \":memory:\" (default)
+  or a filesystem path such as /tmp/test.lbug."
+  [& {:keys [db-path] :or {db-path ":memory:"}}]
+  (graph.ladybug/smoke-test! :db-path db-path))
+
+(defn graph-query-test!
+  "Query Document count for a file's graph db (REPL diagnostic)."
+  [file-id & {:keys [db-path]}]
+  (let [file-id (h/parse-uuid file-id)
+        db-path (or db-path (graph.ladybug/db-path-for-file file-id))
+        stmt    "MATCH (n:Document) RETURN count(n) AS Document_c;"]
+    (graph.ladybug/query-scalar! db-path stmt)))
+
+(defn ingest-file-to-graph!
+  "Project a Penpot file into a per-file Ladybug database.
+
+  Loads and realizes the file from the database, ensures the slice schema,
+  projects Document/Page/shape nodes, and prints a short summary.
+
+  Options:
+  - `:db-path` path or `:memory:`
+  - `:reset-db?` delete any existing db first (default true)
+  - `:skip-stats?` skip post-ingest MATCH count queries (default true)
+  - `:use-arrow?` Arrow bulk load (default true)
+  - `:keep-projection?` keep full `:nodes`/`:edges` in the return value (default false)"
+  [file-id & opts]
+  (let [result (apply graph.ingest/ingest-file! main/system file-id opts)]
+    (graph.report/print-ingest! result)
+    result))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; PROCESSING
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Author	SHA1	Message	Date
Alejandro Alonso	d8c1b0f0d8	WIP arrow api	2026-07-16 12:54:03 +02:00
Alejandro Alonso	d536f4935b	WIP arrow api	2026-07-16 12:44:48 +02:00
Alejandro Alonso	0d5e1c33c0	WIP arrow api	2026-07-16 12:38:32 +02:00
Alejandro Alonso	46937e143a	WIP arrow api	2026-07-16 08:56:03 +02:00
Alejandro Alonso	b3850d39c0	♻️ Derive graph node schema from Malli registry	2026-07-16 07:27:55 +02:00
Alejandro Alonso	b36bbf3bb9	Merge pull request #10672 from penpot/superalex-graph-ingest-slice-1 ✨ Add Penpot-to-Ladybug graph ingest vertical slice	2026-07-15 08:35:40 +02:00
Alejandro Alonso	ad0ad3e75a	🐛 Fix batch delete sync and keep graph console feed alive	2026-07-15 08:34:56 +02:00
Alejandro Alonso	97e339ee5d	✨ Handle mov-objects in debug graph sync	2026-07-15 08:21:58 +02:00
Alejandro Alonso	e2abb1dee1	✨ Incrementally sync debug graph from Penpot file changes	2026-07-15 07:48:40 +02:00
Alejandro Alonso	15dbc5dad2	✨ Add live file-change feed to debug graph console	2026-07-14 13:03:19 +02:00
Alejandro Alonso	726eb440d9	✨ Add debug graph console for in-memory Cypher queries	2026-07-14 12:08:44 +02:00
Alejandro Alonso	445e4970a7	✨ Add Ladybug graph export to debug UI	2026-07-14 11:24:08 +02:00
Alejandro Alonso	0443a8a28f	🐛 Fix graph COPY ingest for multiline text names	2026-07-14 08:27:08 +02:00
Alejandro Alonso	76a0d9ef5e	⚡ Load graph ingest via Ladybug COPY bulk import	2026-07-14 08:01:09 +02:00
Alejandro Alonso	78109b0054	✨ Project nested shapes recursively into the graph	2026-07-14 07:23:54 +02:00
Alejandro Alonso	38488c2612	✨ Validate graph ingest projections with Malli	2026-07-13 18:20:02 +02:00
Alejandro Alonso	275c77471d	♻️ Share Ladybug connection across ingest and stats	2026-07-13 13:43:03 +02:00
Alejandro Alonso	606911ab0d	✨ Use embedded Ladybug Java API instead of CLI	2026-07-13 13:08:52 +02:00
Alejandro Alonso	3714f95666	✨ Add Penpot-to-Ladybug graph ingest vertical slice	2026-07-13 10:58:49 +02:00
Alejandro Alonso	e796512ddf	🎉 Basic lbug connection for ingestion	2026-07-13 07:33:40 +02:00