Initial commit

core/src/main/java/io/xpipe/core/data/DataStructureNode.java

@@ -0,0 +1,75 @@
+package io.xpipe.core.data;
+
+import io.xpipe.core.data.type.DataType;
+
+import java.util.Iterator;
+import java.util.Optional;
+import java.util.Spliterator;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+public abstract class DataStructureNode implements Iterable<DataStructureNode> {
+
+    protected abstract String getName();
+
+    private UnsupportedOperationException unuspported(String s) {
+        return new UnsupportedOperationException(getName() + " does not support " + s);
+    }
+
+    public boolean isTuple() {
+        return false;
+    }
+
+    public boolean isArray() {
+        return false;
+    }
+
+    public boolean isValue() {
+        return false;
+    }
+
+    public int size() {
+        throw unuspported("size computation");
+    }
+
+    public abstract DataType getDataType();
+
+    public DataStructureNode at(int index) {
+        throw unuspported("integer indexing");
+    }
+
+    public DataStructureNode forKey(String name) {
+        throw unuspported("name indexing");
+    }
+
+    public Optional<DataStructureNode> forKeyIfPresent(String name) {
+        throw unuspported("name indexing");
+    }
+
+    public int asInt() {
+        throw unuspported("integer conversion");
+    }
+
+    public String asString() {
+        throw unuspported("string conversion");
+    }
+
+    public Stream<DataStructureNode> stream() {
+        throw unuspported("stream creation");
+    }
+
+    @Override
+    public void forEach(Consumer<? super DataStructureNode> action) {
+        throw unuspported("for each");
+    }
+
+    @Override
+    public Spliterator<DataStructureNode> spliterator() {
+        throw unuspported("spliterator creation");
+    }
+
+    @Override
+    public Iterator<DataStructureNode> iterator() {
+        throw unuspported("iterator creation");
+    }
+}

core/src/main/java/io/xpipe/core/data/DataStructureNodeAcceptor.java

@@ -0,0 +1,6 @@
+package io.xpipe.core.data;
+
+public interface DataStructureNodeAcceptor<T extends DataStructureNode> {
+
+    boolean accept(T node) throws Exception;
+}

core/src/main/java/io/xpipe/core/data/generic/ArrayNode.java

@@ -0,0 +1,71 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+import io.xpipe.core.data.type.ArrayType;
+import io.xpipe.core.data.type.DataType;
+
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+public class ArrayNode extends DataStructureNode {
+
+    private final List<DataStructureNode> valueNodes;
+
+    private ArrayNode(List<DataStructureNode> valueNodes) {
+        this.valueNodes = valueNodes;
+    }
+
+    public static ArrayNode wrap(List<DataStructureNode> valueNodes) {
+        return new ArrayNode(valueNodes);
+    }
+
+    public static ArrayNode copy(List<DataStructureNode> valueNodes) {
+        return new ArrayNode(new ArrayList<>(valueNodes));
+    }
+
+    @Override
+    public Stream<DataStructureNode> stream() {
+        return Collections.unmodifiableList(valueNodes).stream();
+    }
+
+    @Override
+    public boolean isArray() {
+        return true;
+    }
+
+    @Override
+    public int size() {
+        return valueNodes.size();
+    }
+
+    @Override
+    protected String getName() {
+        return "array node";
+    }
+
+    @Override
+    public DataType getDataType() {
+        return ArrayType.of(valueNodes.stream().map(DataStructureNode::getDataType).toList());
+    }
+
+    @Override
+    public DataStructureNode at(int index) {
+        return valueNodes.get(index);
+    }
+
+    @Override
+    public void forEach(Consumer<? super DataStructureNode> action) {
+        valueNodes.forEach(action);
+    }
+
+    @Override
+    public Spliterator<DataStructureNode> spliterator() {
+        return valueNodes.spliterator();
+    }
+
+    @Override
+    public Iterator<DataStructureNode> iterator() {
+        return valueNodes.iterator();
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/ArrayReader.java

@@ -0,0 +1,55 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ArrayReader implements DataStructureNodeReader {
+
+    private final List<DataStructureNode> nodes;
+    private int length;
+    private boolean hasSeenEnd;
+    private int currentIndex = 0;
+    private DataStructureNodeReader currentReader;
+
+    public ArrayReader(int length) {
+        this.length = length;
+        this.nodes = new ArrayList<>(length);
+    }
+
+    @Override
+    public void onArrayStart(String name, int length) {
+        DataStructureNodeReader.super.onArrayStart(name, length);
+    }
+
+    @Override
+    public void onArrayEnd() {
+        DataStructureNodeReader.super.onArrayEnd();
+    }
+
+    @Override
+    public void onTupleStart(String name, int length) {
+        DataStructureNodeReader.super.onTupleStart(name, length);
+    }
+
+    @Override
+    public void onTupleEnd() {
+        DataStructureNodeReader.super.onTupleEnd();
+    }
+
+    @Override
+    public void onValue(String name, byte[] value) {
+        DataStructureNodeReader.super.onValue(name, value);
+    }
+
+    @Override
+    public boolean isDone() {
+        return false;
+    }
+
+    @Override
+    public DataStructureNode create() {
+        return null;
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/DataStreamCallback.java

@@ -0,0 +1,30 @@
+package io.xpipe.core.data.generic;
+
+import java.util.function.Consumer;
+
+public interface DataStreamCallback {
+
+    static DataStreamCallback flat(Consumer<byte[]> con) {
+        return new DataStreamCallback() {
+            @Override
+            public void onValue(String name, byte[] value) {
+                con.accept(value);
+            }
+        };
+    }
+
+    default void onArrayStart(String name, int length) {
+    }
+
+    default void onArrayEnd() {
+    }
+
+    default void onTupleStart(String name, int length) {
+    }
+
+    default void onTupleEnd() {
+    }
+
+    default void onValue(String name, byte[] value) {
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/DataStreamReader.java

@@ -0,0 +1,59 @@
+package io.xpipe.core.data.generic;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class DataStreamReader {
+
+    private static final int TUPLE_ID = 1;
+    private static final int ARRAY_ID = 2;
+    private static final int VALUE_ID = 3;
+
+    public static void read(InputStream in, DataStreamCallback cb) throws IOException {
+        var b = in.read();
+        switch (b) {
+            case TUPLE_ID -> {
+                readTuple(in, cb);
+            }
+            case ARRAY_ID -> {
+                readArray(in, cb);
+            }
+            case VALUE_ID -> {
+                readValue(in, cb);
+            }
+            default -> throw new IllegalStateException("Unexpected value: " + b);
+        }
+    }
+
+    private static String readName(InputStream in) throws IOException {
+        var nameLength = in.read();
+        return new String(in.readNBytes(nameLength));
+    }
+
+    private static void readTuple(InputStream in, DataStreamCallback cb) throws IOException {
+        var name = readName(in);
+        var size = in.read();
+        cb.onTupleStart(name, size);
+        for (int i = 0; i < size; i++) {
+            read(in, cb);
+        }
+        cb.onTupleEnd();
+    }
+
+    private static void readArray(InputStream in, DataStreamCallback cb) throws IOException {
+        var name = readName(in);
+        var size = in.read();
+        cb.onArrayStart(name, size);
+        for (int i = 0; i < size; i++) {
+            read(in, cb);
+        }
+        cb.onArrayEnd();
+    }
+
+    private static void readValue(InputStream in, DataStreamCallback cb) throws IOException {
+        var name = readName(in);
+        var size = in.read();
+        var data = in.readNBytes(size);
+        cb.onValue(name, data);
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/DataStreamWriter.java

@@ -0,0 +1,33 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+public class DataStreamWriter {
+
+    private static final int TUPLE_ID = 1;
+    private static final int ARRAY_ID = 2;
+    private static final int VALUE_ID = 3;
+
+    public static void write(OutputStream out, DataStructureNode node) throws IOException {
+        if (node.isTuple()) {
+            writeTuple(out, (TupleNode) node);
+        }
+    }
+
+    private static void writeName(OutputStream out, String s) throws IOException {
+        out.write(s.length());
+        out.write(s.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private static void writeTuple(OutputStream out, TupleNode tuple) throws IOException {
+        out.write(TUPLE_ID);
+        for (int i = 0; i < tuple.size(); i++) {
+            writeName(out, tuple.nameAt(i));
+            write(out, tuple.at(i));
+        }
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/DataStructureNodePointer.java

@@ -0,0 +1,239 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public class DataStructureNodePointer {
+
+    private final List<Element> path;
+
+    public DataStructureNodePointer(List<Element> path) {
+        this.path = path;
+
+        if (path.size() == 0) {
+            throw new IllegalArgumentException();
+        }
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static Builder fromBase(DataStructureNodePointer pointer) {
+        return new Builder(pointer);
+    }
+
+    public String toString() {
+        return "/" + path.stream().map(Element::toString).collect(Collectors.joining("/"));
+    }
+
+    public int size() {
+        return path.size();
+    }
+
+    public boolean isValid(DataStructureNode input) {
+        return get(input) != null;
+    }
+
+    public DataStructureNode get(DataStructureNode root) {
+        DataStructureNode current = root;
+        for (Element value : path) {
+            var found = value.tryMatch(current);
+            if (found == null) {
+                return null;
+            } else {
+                current = found;
+            }
+        }
+        return current;
+    }
+
+    public Optional<DataStructureNode> getIfPresent(DataStructureNode root) {
+        return Optional.ofNullable(get(root));
+    }
+
+    public List<Element> getPath() {
+        return path;
+    }
+
+    public static interface Element {
+
+        DataStructureNode tryMatch(DataStructureNode n);
+
+        default String getKey(DataStructureNode n) {
+            return null;
+        }
+    }
+
+    public static final record NameElement(String name) implements Element {
+
+        @Override
+        public DataStructureNode tryMatch(DataStructureNode n) {
+            return n.forKeyIfPresent(name).orElse(null);
+        }
+
+        @Override
+        public String getKey(DataStructureNode n) {
+            return name;
+        }
+
+        @Override
+        public String toString() {
+            return name;
+        }
+    }
+
+    public static final record IndexElement(int index) implements Element {
+
+        @Override
+        public DataStructureNode tryMatch(DataStructureNode n) {
+            if (n.size() > index) {
+                return n.at(index);
+            }
+            return null;
+        }
+
+        @Override
+        public String toString() {
+            return "[" + index + "]";
+        }
+    }
+
+    public static final record SupplierElement(Supplier<String> keySupplier) implements Element {
+
+        @Override
+        public DataStructureNode tryMatch(DataStructureNode n) {
+            var name = keySupplier.get();
+            if (name != null) {
+                return n.forKeyIfPresent(name).orElse(null);
+            }
+            return null;
+        }
+
+        @Override
+        public String getKey(DataStructureNode n) {
+            return keySupplier.get();
+        }
+
+        @Override
+        public String toString() {
+            return "[$s]";
+        }
+    }
+
+    public static final record FunctionElement(Function<DataStructureNode, String> keyFunc) implements Element {
+
+        @Override
+        public DataStructureNode tryMatch(DataStructureNode n) {
+            var name = keyFunc.apply(n);
+            if (name != null) {
+                return n.forKeyIfPresent(name).orElse(null);
+            }
+            return null;
+        }
+
+        @Override
+        public String getKey(DataStructureNode n) {
+            return keyFunc.apply(n);
+        }
+
+        @Override
+        public String toString() {
+            return "[$s]";
+        }
+    }
+
+    public static final record SelectorElement(Predicate<DataStructureNode> selector) implements Element {
+
+        @Override
+        public DataStructureNode tryMatch(DataStructureNode n) {
+            var res = n.stream()
+                    .filter(selector)
+                    .findAny();
+            return res.orElse(null);
+        }
+
+        @Override
+        public String toString() {
+            return "[$(...)]";
+        }
+    }
+
+    public static class Builder {
+
+        private final List<Element> path;
+
+        public Builder() {
+            this.path = new ArrayList<>();
+        }
+
+        private Builder(List<Element> path) {
+            this.path = path;
+        }
+
+        public Builder(DataStructureNodePointer pointer) {
+            this.path = new ArrayList<>(pointer.path);
+        }
+
+        public Builder copy() {
+            return new Builder(new ArrayList<>(path));
+        }
+
+
+        public Builder name(String name) {
+            path.add(new NameElement(name));
+            return this;
+        }
+
+        public Builder index(int index) {
+            path.add(new IndexElement(index));
+            return this;
+        }
+
+        public Builder supplier(Supplier<String> keySupplier) {
+            path.add(new SupplierElement(keySupplier));
+            return this;
+        }
+
+        public Builder function(Function<DataStructureNode, String> keyFunc) {
+            path.add(new FunctionElement(keyFunc));
+            return this;
+        }
+
+        public Builder selector(Predicate<DataStructureNode> selector) {
+            path.add(new SelectorElement(selector));
+            return this;
+        }
+
+        public Builder pointerEvaluation(DataStructureNodePointer pointer) {
+            return pointerEvaluation(pointer, n -> {
+                if (!n.isValue()) {
+                    return null;
+                }
+                return n.asString();
+            });
+        }
+
+        public Builder pointerEvaluation(DataStructureNodePointer pointer, Function<DataStructureNode, String> converter) {
+            path.add(new FunctionElement((current) -> {
+                var res = pointer.get(current);
+                if (res != null) {
+                    return converter.apply(res);
+                }
+                return null;
+            }));
+            return this;
+        }
+
+        public DataStructureNodePointer build() {
+            return new DataStructureNodePointer(path);
+        }
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/DataStructureNodeReader.java

@@ -0,0 +1,10 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+public interface DataStructureNodeReader extends DataStreamCallback {
+
+    boolean isDone();
+
+    DataStructureNode create();
+}

core/src/main/java/io/xpipe/core/data/generic/DataStructureReader.java

@@ -0,0 +1,69 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+public class DataStructureReader implements DataStreamCallback {
+
+    private boolean isWrapped;
+    private DataStructureNodeReader reader;
+
+    public DataStructureNode create() {
+        return null;
+    }
+
+
+    @Override
+    public void onArrayStart(String name, int length) {
+        if (reader != null) {
+            reader.onArrayStart(name, length);
+            return;
+        }
+
+        if (name != null) {
+            reader = new TupleReader(1);
+            reader.onArrayStart(name, length);
+        } else {
+            reader = new ArrayReader(length);
+            reader.onArrayStart(null, length);
+        }
+    }
+
+    @Override
+    public void onArrayEnd() {
+        if (reader != null) {
+            reader.onArrayEnd();
+        }
+    }
+
+    @Override
+    public void onTupleStart(String name, int length) {
+        if (reader != null) {
+            reader.onTupleStart(name, length);
+            return;
+        }
+
+        if (name != null) {
+            reader = new TupleReader(1);
+            reader.onTupleStart(name, length);
+        } else {
+            reader = new TupleReader(length);
+        }
+    }
+
+    @Override
+    public void onTupleEnd() {
+        if (reader != null) {
+            reader.onTupleEnd();
+            if (reader.isDone()) {
+
+            }
+        }
+
+        DataStreamCallback.super.onTupleEnd();
+    }
+
+    @Override
+    public void onValue(String name, byte[] value) {
+        DataStreamCallback.super.onValue(name, value);
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/TupleNode.java

@@ -0,0 +1,79 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+import io.xpipe.core.data.type.DataType;
+import io.xpipe.core.data.type.TupleType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+public class TupleNode extends DataStructureNode {
+
+    private final List<String> names;
+    private final List<DataStructureNode> nodes;
+
+    private TupleNode(List<String> names, List<DataStructureNode> nodes) {
+        this.names = names;
+        this.nodes = nodes;
+    }
+
+    public static TupleNode wrap(List<String> names, List<DataStructureNode> nodes) {
+        return new TupleNode(names, nodes);
+    }
+
+    public static TupleNode copy(List<String> names, List<DataStructureNode> nodes) {
+        return new TupleNode(new ArrayList<>(names), new ArrayList<>(nodes));
+    }
+
+    public boolean isTuple() {
+        return true;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return TupleType.wrap(names, nodes.stream().map(DataStructureNode::getDataType).toList());
+    }
+
+    @Override
+    protected String getName() {
+        return "tuple node";
+    }
+
+    @Override
+    public DataStructureNode at(int index) {
+        return nodes.get(index);
+    }
+
+    @Override
+    public DataStructureNode forKey(String name) {
+        return nodes.get(names.indexOf(name));
+    }
+
+    @Override
+    public Optional<DataStructureNode> forKeyIfPresent(String name) {
+        if (!names.contains(name)) {
+            return Optional.empty();
+        }
+
+        return Optional.of(nodes.get(names.indexOf(name)));
+    }
+
+    @Override
+    public int size() {
+        return nodes.size();
+    }
+
+    public String nameAt(int index) {
+        return names.get(index);
+    }
+
+    public List<String> getNames() {
+        return Collections.unmodifiableList(names);
+    }
+
+    public List<DataStructureNode> getNodes() {
+        return Collections.unmodifiableList(nodes);
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/TupleReader.java

@@ -0,0 +1,119 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TupleReader implements DataStructureNodeReader {
+
+    private final int length;
+    private final List<String> names;
+    private final List<DataStructureNode> nodes;
+    private boolean hasSeenEnd;
+    private int currentIndex = 0;
+    private DataStructureNodeReader currentReader;
+
+    public TupleReader(int length) {
+        this.length = length;
+        this.names = new ArrayList<>(length);
+        this.nodes = new ArrayList<>(length);
+    }
+
+    private void put(String name, DataStructureNode node) {
+        this.names.add(name);
+        this.nodes.add(node);
+        currentIndex++;
+    }
+
+    private void putNode(DataStructureNode node) {
+        this.nodes.add(node);
+        currentIndex++;
+    }
+
+    private boolean filled() {
+        return currentIndex == length;
+    }
+
+    @Override
+    public void onArrayStart(String name, int length) {
+        if (currentReader != null) {
+            currentReader.onArrayStart(name, length);
+            return;
+        }
+
+        names.add(name);
+        currentReader = new ArrayReader(length);
+    }
+
+    @Override
+    public void onArrayEnd() {
+        if (currentReader != null) {
+            currentReader.onArrayEnd();
+            if (currentReader.isDone()) {
+                putNode(currentReader.create());
+                currentReader = null;
+            }
+            return;
+        }
+
+        throw new IllegalStateException();
+    }
+
+    @Override
+    public void onTupleStart(String name, int length) {
+        if (currentReader != null) {
+            currentReader.onTupleStart(name, length);
+            return;
+        }
+
+        names.add(name);
+        currentReader = new TupleReader(length);
+    }
+
+    @Override
+    public void onTupleEnd() {
+        if (currentReader != null) {
+            currentReader.onTupleEnd();
+            if (currentReader.isDone()) {
+                putNode(currentReader.create());
+                currentReader = null;
+            }
+            return;
+        }
+
+        if (!filled()) {
+            throw new IllegalStateException();
+        }
+
+        hasSeenEnd = true;
+    }
+
+    @Override
+    public void onValue(String name, byte[] value) {
+        if (currentReader != null) {
+            currentReader.onValue(name, value);
+            return;
+        }
+
+        if (filled()) {
+            throw new IllegalStateException();
+        }
+
+        put(name, ValueNode.wrap(value));
+    }
+
+    @Override
+    public boolean isDone() {
+        return filled() && hasSeenEnd;
+    }
+
+    @Override
+    public DataStructureNode create() {
+        if (!isDone()) {
+            throw new IllegalStateException();
+        }
+
+        return TupleNode.wrap(names, nodes);
+    }
+}

core/src/main/java/io/xpipe/core/data/generic/ValueNode.java

@@ -0,0 +1,47 @@
+package io.xpipe.core.data.generic;
+
+import io.xpipe.core.data.DataStructureNode;
+import io.xpipe.core.data.type.DataType;
+import io.xpipe.core.data.type.ValueType;
+
+public class ValueNode extends DataStructureNode {
+
+    private final byte[] data;
+
+    private ValueNode(byte[] data) {
+        this.data = data;
+    }
+
+    public static ValueNode wrap(byte[] data) {
+        return new ValueNode(data);
+    }
+
+    @Override
+    public boolean isValue() {
+        return true;
+    }
+
+    @Override
+    public int asInt() {
+        return Integer.parseInt(asString());
+    }
+
+    @Override
+    public String asString() {
+        return new String(data);
+    }
+
+    @Override
+    protected String getName() {
+        return "value node";
+    }
+
+    @Override
+    public DataType getDataType() {
+        return new ValueType();
+    }
+
+    public byte[] getRawData() {
+        return data;
+    }
+}

core/src/main/java/io/xpipe/core/data/type/ArrayType.java

@@ -0,0 +1,58 @@
+package io.xpipe.core.data.type;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.xpipe.core.data.type.callback.DataTypeCallback;
+
+import java.util.List;
+
+@JsonTypeName("array")
+public class ArrayType implements DataType {
+
+    public static ArrayType of(List<DataType> types) {
+        if (types.size() == 0) {
+            return new ArrayType(null);
+        }
+
+        var first = types.get(0);
+        var eq = types.stream().allMatch(d -> d.equals(first));
+        return new ArrayType(eq ? first : null);
+    }
+
+    private final DataType sharedType;
+
+    public ArrayType(DataType sharedType) {
+        this.sharedType = sharedType;
+    }
+
+    public boolean isSimple() {
+        return hasSharedType() && getSharedType().isValue();
+    }
+
+    public boolean hasSharedType() {
+        return sharedType != null;
+    }
+
+    public DataType getSharedType() {
+        return sharedType;
+    }
+
+    @Override
+    public boolean isTuple() {
+        return false;
+    }
+
+    @Override
+    public boolean isArray() {
+        return true;
+    }
+
+    @Override
+    public boolean isValue() {
+        return false;
+    }
+
+    @Override
+    public void traverseType(DataTypeCallback cb) {
+        cb.onArray(this);
+    }
+}

core/src/main/java/io/xpipe/core/data/type/DataType.java

@@ -0,0 +1,16 @@
+package io.xpipe.core.data.type;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import io.xpipe.core.data.type.callback.DataTypeCallback;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+public interface DataType {
+
+    boolean isTuple();
+
+    boolean isArray();
+
+    boolean isValue();
+
+    void traverseType(DataTypeCallback cb);
+}

core/src/main/java/io/xpipe/core/data/type/TupleType.java

@@ -0,0 +1,69 @@
+package io.xpipe.core.data.type;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.xpipe.core.data.type.callback.DataTypeCallback;
+
+import java.util.Collections;
+import java.util.List;
+
+@JsonTypeName("tuple")
+public class TupleType implements DataType {
+
+    private List<String> names;
+    private List<DataType> types;
+
+    @JsonCreator
+    private TupleType(List<String> names, List<DataType> types) {
+        this.names = names;
+        this.types = types;
+    }
+
+    public static TupleType empty() {
+        return new TupleType(List.of(), List.of());
+    }
+
+    public static TupleType wrap(List<String> names, List<DataType> types) {
+        return new TupleType(names, types);
+    }
+
+    public static TupleType wrapWithoutNames(List<DataType> types) {
+        return new TupleType(Collections.nCopies(types.size(), null), types);
+    }
+
+    @Override
+    public boolean isTuple() {
+        return true;
+    }
+
+    @Override
+    public boolean isArray() {
+        return false;
+    }
+
+    @Override
+    public boolean isValue() {
+        return false;
+    }
+
+    @Override
+    public void traverseType(DataTypeCallback cb) {
+        cb.onTupleBegin(this);
+        for (var t : types) {
+            t.traverseType(cb);
+        }
+        cb.onTupleEnd();
+    }
+
+    public int getSize() {
+        return types.size();
+    }
+
+    public List<String> getNames() {
+        return names;
+    }
+
+    public List<DataType> getTypes() {
+        return types;
+    }
+}

core/src/main/java/io/xpipe/core/data/type/TypedDataStreamReader.java

@@ -0,0 +1,95 @@
+package io.xpipe.core.data.type;
+
+import io.xpipe.core.data.type.callback.TypedDataStreamCallback;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class TypedDataStreamReader {
+
+    private static final int STRUCTURE_ID = 0;
+    private static final int TUPLE_ID = 1;
+    private static final int ARRAY_ID = 2;
+    private static final int VALUE_ID = 3;
+
+    public static boolean hasNext(InputStream in) throws IOException {
+        var b = in.read();
+        if (b == -1) {
+            return false;
+        }
+
+        if (b != STRUCTURE_ID) {
+            throw new IOException("Unexpected value: " + b);
+        }
+
+        return true;
+    }
+
+    public static void readStructures(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        while (true) {
+            if (!hasNext(in)) {
+                break;
+            }
+
+            cb.onNodeBegin();
+            read(in, cb);
+            cb.onNodeEnd();
+        }
+    }
+
+    public static void readStructure(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        if (!hasNext(in)) {
+            throw new IllegalStateException("No structure to read");
+        }
+
+        cb.onNodeBegin();
+        read(in, cb);
+        cb.onNodeEnd();
+    }
+
+    private static void read(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        var b = in.read();
+
+        // Skip
+        if (b == STRUCTURE_ID) {
+            b = in.read();
+        }
+
+        switch (b) {
+            case TUPLE_ID -> {
+                readTuple(in, cb);
+            }
+            case ARRAY_ID -> {
+                readArray(in, cb);
+            }
+            case VALUE_ID -> {
+                readValue(in, cb);
+            }
+            default -> throw new IllegalStateException("Unexpected value: " + b);
+        }
+    }
+
+    private static void readTuple(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        var size = in.read();
+        cb.onTupleBegin(size);
+        for (int i = 0; i < size; i++) {
+            read(in, cb);
+        }
+        cb.onTupleEnd();
+    }
+
+    private static void readArray(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        var size = in.read();
+        cb.onArrayBegin(size);
+        for (int i = 0; i < size; i++) {
+            read(in, cb);
+        }
+        cb.onArrayEnd();
+    }
+
+    private static void readValue(InputStream in, TypedDataStreamCallback cb) throws IOException {
+        var size = in.read();
+        var data = in.readNBytes(size);
+        cb.onValue(data);
+    }
+}

core/src/main/java/io/xpipe/core/data/type/TypedDataStreamWriter.java

@@ -0,0 +1,58 @@
+package io.xpipe.core.data.type;
+
+import io.xpipe.core.data.DataStructureNode;
+import io.xpipe.core.data.generic.ArrayNode;
+import io.xpipe.core.data.generic.TupleNode;
+import io.xpipe.core.data.generic.ValueNode;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class TypedDataStreamWriter {
+
+    private static final int STRUCTURE_ID = 0;
+    private static final int TUPLE_ID = 1;
+    private static final int ARRAY_ID = 2;
+    private static final int VALUE_ID = 3;
+
+    public static void writeStructure(OutputStream out, DataStructureNode node) throws IOException {
+        out.write(STRUCTURE_ID);
+        write(out, node);
+    }
+
+    private static void write(OutputStream out, DataStructureNode node) throws IOException {
+        if (node.isTuple()) {
+            writeTuple(out, (TupleNode) node);
+        }
+        else if (node.isArray()) {
+            writeArray(out, (ArrayNode) node);
+        }
+        else if (node.isValue()) {
+            writeValue(out, (ValueNode) node);
+        } else {
+            throw new AssertionError();
+        }
+    }
+
+    private static void writeValue(OutputStream out, ValueNode n) throws IOException {
+        out.write(VALUE_ID);
+        out.write(n.getRawData().length);
+        out.write(n.getRawData());
+    }
+
+    private static void writeTuple(OutputStream out, TupleNode tuple) throws IOException {
+        out.write(TUPLE_ID);
+        out.write(tuple.size());
+        for (int i = 0; i < tuple.size(); i++) {
+            write(out, tuple.at(i));
+        }
+    }
+
+    private static void writeArray(OutputStream out, ArrayNode array) throws IOException {
+        out.write(ARRAY_ID);
+        out.write(array.size());
+        for (int i = 0; i < array.size(); i++) {
+            write(out, array.at(i));
+        }
+    }
+}

core/src/main/java/io/xpipe/core/data/type/ValueType.java

@@ -0,0 +1,30 @@
+package io.xpipe.core.data.type;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.xpipe.core.data.type.callback.DataTypeCallback;
+
+@JsonTypeName("value")
+public class ValueType implements DataType {
+
+    @Override
+    public boolean isTuple() {
+        return false;
+    }
+
+    @JsonIgnore
+    @Override
+    public boolean isArray() {
+        return false;
+    }
+
+    @Override
+    public boolean isValue() {
+        return true;
+    }
+
+    @Override
+    public void traverseType(DataTypeCallback cb) {
+        cb.onValue();
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/DataTypeCallback.java

@@ -0,0 +1,42 @@
+package io.xpipe.core.data.type.callback;
+
+import io.xpipe.core.data.type.ArrayType;
+import io.xpipe.core.data.type.DataType;
+import io.xpipe.core.data.type.TupleType;
+import io.xpipe.core.data.type.ValueType;
+
+import java.util.function.Consumer;
+
+public interface DataTypeCallback {
+
+    public static DataTypeCallback flatten(Consumer<DataType> typeConsumer) {
+        return new DataTypeCallback() {
+            @Override
+            public void onValue() {
+                typeConsumer.accept(new ValueType());
+            }
+
+            @Override
+            public void onTupleBegin(TupleType tuple) {
+                typeConsumer.accept(tuple);
+            }
+
+            @Override
+            public void onArray(ArrayType type) {
+                typeConsumer.accept(type);
+            }
+        };
+    }
+
+    default void onValue() {
+    }
+
+    default void onTupleBegin(TupleType tuple) {
+    }
+
+    default void onTupleEnd() {
+    }
+
+    default void onArray(ArrayType type) {
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/DataTypeCallbacks.java

@@ -0,0 +1,61 @@
+package io.xpipe.core.data.type.callback;
+
+import io.xpipe.core.data.generic.DataStructureNodePointer;
+import io.xpipe.core.data.type.ArrayType;
+import io.xpipe.core.data.type.TupleType;
+
+import java.util.Stack;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public class DataTypeCallbacks {
+
+    public static DataTypeCallback visitTuples(Consumer<String> newTuple, Runnable endTuple, BiConsumer<String, DataStructureNodePointer> newValue) {
+        return new DataTypeCallback() {
+
+            private final Stack<String> keyNames = new Stack<>();
+            private final Stack<DataStructureNodePointer.Builder> builders = new Stack<>();
+
+            {
+                builders.push(DataStructureNodePointer.builder());
+            }
+
+            private boolean isOnTopLevel() {
+                return keyNames.size() == 0;
+            }
+
+            @Override
+            public void onTupleBegin(TupleType tuple) {
+                if (!isOnTopLevel()) {
+                    newTuple.accept(keyNames.peek());
+                }
+                tuple.getNames().forEach(n -> {
+                    keyNames.push(n);
+                    builders.push(builders.peek().copy().name(n));
+                    tuple.getTypes().forEach(dt -> dt.traverseType(this));
+                });
+            }
+
+            @Override
+            public void onValue() {
+                newValue.accept(keyNames.peek(), builders.peek().build());
+                keyNames.pop();
+                builders.pop();
+            }
+
+            @Override
+            public void onTupleEnd() {
+                endTuple.run();
+            }
+
+            @Override
+            public void onArray(ArrayType type) {
+                if (!type.isSimple()) {
+                    throw new IllegalStateException();
+                }
+
+                newValue.accept(keyNames.peek(), builders.peek().build());
+            }
+        };
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/FlatArrayTypeCallback.java

@@ -0,0 +1,63 @@
+package io.xpipe.core.data.type.callback;
+
+import io.xpipe.core.data.type.TupleType;
+
+public class FlatArrayTypeCallback implements DataTypeCallback {
+
+    private final FlatCallback cb;
+    private int arrayDepth = 0;
+
+    public FlatArrayTypeCallback(FlatCallback cb) {
+        this.cb = cb;
+    }
+
+    private boolean isInArray() {
+        return arrayDepth > 0;
+    }
+
+    @Override
+    public void onValue() {
+        if (isInArray()) {
+            return;
+        }
+
+        cb.onValue();
+    }
+
+    @Override
+    public void onTupleBegin(TupleType tuple) {
+        if (isInArray()) {
+            throw new IllegalStateException();
+        }
+
+        cb.onTupleBegin(tuple);
+    }
+
+    @Override
+    public void onTupleEnd() {
+        cb.onTupleEnd();
+    }
+
+    public void onArray() {
+        if (isInArray()) {
+            throw new IllegalStateException();
+        }
+
+        arrayDepth++;
+    }
+
+    public static interface FlatCallback {
+
+        default void onValue() {
+        }
+
+        default void onTupleBegin(TupleType tuple) {
+        }
+
+        default void onTupleEnd() {
+        }
+
+        default void onFlatArray() {
+        }
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/ReusableTypedDataStructureNodeCallback.java

@@ -0,0 +1,29 @@
+package io.xpipe.core.data.type.callback;
+
+public class ReusableTypedDataStructureNodeCallback implements TypedDataStreamCallback {
+
+    @Override
+    public void onValue(byte[] data) {
+        TypedDataStreamCallback.super.onValue(data);
+    }
+
+    @Override
+    public void onTupleBegin(int size) {
+        TypedDataStreamCallback.super.onTupleBegin(size);
+    }
+
+    @Override
+    public void onTupleEnd() {
+        TypedDataStreamCallback.super.onTupleEnd();
+    }
+
+    @Override
+    public void onArrayBegin(int size) {
+        TypedDataStreamCallback.super.onArrayBegin(size);
+    }
+
+    @Override
+    public void onArrayEnd() {
+        TypedDataStreamCallback.super.onArrayEnd();
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/TableTypeCallback.java

@@ -0,0 +1,80 @@
+package io.xpipe.core.data.type.callback;
+
+import io.xpipe.core.data.generic.DataStructureNodePointer;
+import io.xpipe.core.data.type.ArrayType;
+import io.xpipe.core.data.type.TupleType;
+
+import java.util.Stack;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public class TableTypeCallback implements DataTypeCallback {
+
+    private final Stack<TupleType> tuples = new Stack<>();
+    private final Stack<Integer> keyIndices = new Stack<>();
+    private final Consumer<String> newTuple;
+    private final Runnable endTuple;
+    private final BiConsumer<String, DataStructureNodePointer> newValue;
+
+    private TableTypeCallback(Consumer<String> newTuple, Runnable endTuple, BiConsumer<String, DataStructureNodePointer> newValue) {
+        this.newTuple = newTuple;
+        this.endTuple = endTuple;
+        this.newValue = newValue;
+    }
+
+    public static DataTypeCallback create(Consumer<String> newTuple, Runnable endTuple, BiConsumer<String, DataStructureNodePointer> newValue) {
+        return new TableTypeCallback(newTuple, endTuple, newValue);
+    }
+
+    private boolean isOnTopLevel() {
+        return tuples.size() <= 1;
+    }
+
+    private void onAnyValue() {
+        var pointer = DataStructureNodePointer.builder();
+        for (int index : keyIndices) {
+            pointer.index(index);
+        }
+        var p = pointer.build();
+        newValue.accept(tuples.peek().getNames().get(keyIndices.peek()), p);
+
+        moveIndex();
+    }
+
+    private void moveIndex() {
+        var index = keyIndices.pop();
+        index++;
+        keyIndices.push(index);
+    }
+
+    @Override
+    public void onValue() {
+        onAnyValue();
+    }
+
+    @Override
+    public void onTupleBegin(TupleType tuple) {
+        if (!isOnTopLevel()) {
+            moveIndex();
+        }
+
+        tuples.push(tuple);
+        keyIndices.push(0);
+
+        if (!isOnTopLevel()) {
+            newTuple.accept(tuples.peek().getNames().get(keyIndices.peek()));
+        }
+    }
+
+    @Override
+    public void onTupleEnd() {
+        endTuple.run();
+        tuples.pop();
+        keyIndices.pop();
+    }
+
+    @Override
+    public void onArray(ArrayType type) {
+        onAnyValue();
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/TypedDataStreamCallback.java

@@ -0,0 +1,25 @@
+package io.xpipe.core.data.type.callback;
+
+public interface TypedDataStreamCallback {
+
+    default void onValue(byte[] data) {
+    }
+
+    default void onTupleBegin(int size) {
+    }
+
+    default void onTupleEnd() {
+    }
+
+    default void onArrayBegin(int size) {
+    }
+
+    default void onArrayEnd() {
+    }
+
+    default void onNodeBegin() {
+    }
+
+    default void onNodeEnd() {
+    }
+}

core/src/main/java/io/xpipe/core/data/type/callback/TypedDataStructureNodeCallback.java

@@ -0,0 +1,132 @@
+package io.xpipe.core.data.type.callback;
+
+import io.xpipe.core.data.DataStructureNode;
+import io.xpipe.core.data.generic.ArrayNode;
+import io.xpipe.core.data.generic.TupleNode;
+import io.xpipe.core.data.generic.ValueNode;
+import io.xpipe.core.data.type.DataType;
+import io.xpipe.core.data.type.TupleType;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
+import java.util.function.Consumer;
+
+public class TypedDataStructureNodeCallback implements TypedDataStreamCallback {
+
+    private final List<DataType> flattened;
+    private int dataTypeIndex;
+    private Stack<List<DataStructureNode>> children;
+    private Stack<DataStructureNode> nodes;
+    private DataStructureNode readNode;
+    private final Consumer<DataStructureNode> consumer;
+
+    public TypedDataStructureNodeCallback(DataType type, Consumer<DataStructureNode> consumer) {
+        this.consumer = consumer;
+        flattened = new ArrayList<>();
+        children = new Stack<>();
+        nodes = new Stack<>();
+        type.traverseType(DataTypeCallback.flatten(d -> flattened.add(d)));
+    }
+
+    @Override
+    public void onNodeBegin() {
+        if (nodes.size() != 0 || children.size() != 0) {
+            throw new IllegalStateException();
+        }
+
+        dataTypeIndex = 0;
+        readNode = null;
+    }
+
+    @Override
+    public void onNodeEnd() {
+        if (nodes.size() != 0 || children.size() != 0 || readNode == null) {
+            throw new IllegalStateException();
+        }
+
+        consumer.accept(readNode);
+    }
+
+    @Override
+    public void onValue(byte[] data) {
+        children.peek().add(ValueNode.wrap(data));
+        if (!flattened.get(dataTypeIndex).isArray()) {
+            dataTypeIndex++;
+        }
+    }
+
+    protected void newTuple() {
+        TupleType tupleType = (TupleType) flattened.get(dataTypeIndex);
+        var l = new ArrayList<DataStructureNode>(tupleType.getSize());
+        children.push(l);
+        var newNode = TupleNode.wrap(tupleType.getNames(), l);
+        nodes.push(newNode);
+    }
+
+    protected void newArray() {
+        var l = new ArrayList<DataStructureNode>();
+        children.push(new ArrayList<>());
+        var newNode = ArrayNode.wrap(l);
+        nodes.push(newNode);
+    }
+
+    private void finishTuple() {
+        children.pop();
+        dataTypeIndex++;
+        var popped = nodes.pop();
+        if (!popped.isTuple()) {
+            throw new IllegalStateException();
+        }
+
+        TupleNode tuple = (TupleNode) popped;
+        if (tuple.getNames().size() != tuple.getNodes().size()) {
+            throw new IllegalStateException("");
+        }
+
+        if (nodes.empty()) {
+            readNode = popped;
+        } else {
+            children.peek().add(popped);
+        }
+    }
+
+    private void finishArray() {
+        children.pop();
+        dataTypeIndex++;
+        var popped = nodes.pop();
+        if (nodes.empty()) {
+            readNode = popped;
+        } else {
+            children.peek().add(popped);
+        }
+    }
+
+    @Override
+    public void onTupleBegin(int size) {
+        if (!flattened.get(dataTypeIndex).isTuple()) {
+            throw new IllegalStateException();
+        }
+
+        newTuple();
+    }
+
+    @Override
+    public void onTupleEnd() {
+        finishTuple();
+    }
+
+    @Override
+    public void onArrayBegin(int size) {
+        if (!flattened.get(dataTypeIndex).isArray()) {
+            throw new IllegalStateException();
+        }
+
+        newArray();
+    }
+
+    @Override
+    public void onArrayEnd() {
+        finishArray();
+    }
+}

core/src/main/java/io/xpipe/core/source/DataSource.java

@@ -0,0 +1,14 @@
+package io.xpipe.core.source;
+
+import io.xpipe.core.store.DataStore;
+
+import java.util.Optional;
+
+public interface DataSource<DS extends DataStore> {
+
+    default Optional<String> determineDefaultName(DS store) {
+        return Optional.empty();
+    }
+
+    DataSourceType getType();
+}

core/src/main/java/io/xpipe/core/source/DataSourceConnection.java

@@ -0,0 +1,8 @@
+package io.xpipe.core.source;
+
+public interface DataSourceConnection extends AutoCloseable {
+
+    void init() throws Exception;
+
+    void close() throws Exception;
+}

core/src/main/java/io/xpipe/core/source/DataSourceId.java

@@ -0,0 +1,55 @@
+package io.xpipe.core.source;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public class DataSourceId {
+
+    public static final char SEPARATOR = ':';
+
+    private final String collectionName;
+    private final String entryName;
+
+    @JsonCreator
+    public DataSourceId(String collectionName, String entryName) {
+        this.collectionName = collectionName;
+        this.entryName = entryName;
+    }
+
+    public DataSourceId withEntryName(String newName) {
+        return new DataSourceId(collectionName, newName);
+    }
+
+    public static DataSourceId fromString(String s) {
+        var split = s.split(String.valueOf(SEPARATOR));
+        if (split.length != 2) {
+            throw new IllegalArgumentException();
+        }
+
+        if (split[1].length() == 0) {
+            throw new IllegalArgumentException();
+        }
+
+        return new DataSourceId(split[0].length() > 0 ? split[0] : null, split[1]);
+    }
+
+    public boolean hasCollection() {
+        return collectionName != null;
+    }
+
+    @Override
+    public String toString() {
+        return (collectionName != null ? collectionName : "") + SEPARATOR + entryName;
+    }
+
+    public String toReferenceValue() {
+        return toString().toLowerCase();
+    }
+
+    public String getCollectionName() {
+        return collectionName;
+    }
+
+    public String getEntryName() {
+        return entryName;
+    }
+}

core/src/main/java/io/xpipe/core/source/DataSourceType.java

@@ -0,0 +1,12 @@
+package io.xpipe.core.source;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public enum DataSourceType {
+
+    @JsonProperty("table")
+    TABLE,
+
+    @JsonProperty("structure")
+    STRUCTURE
+}

core/src/main/java/io/xpipe/core/source/DataStructureConnection.java

@@ -0,0 +1,5 @@
+package io.xpipe.core.source;
+
+public interface DataStructureConnection extends DataSourceConnection {
+
+}

core/src/main/java/io/xpipe/core/source/DataStructureSource.java

@@ -0,0 +1,11 @@
+package io.xpipe.core.source;
+
+public abstract class DataStructureSource implements DataSource {
+
+    public abstract DataSourceConnection openConnection() throws Exception;
+
+    @Override
+    public DataSourceType getType() {
+        return DataSourceType.STRUCTURE;
+    }
+}

core/src/main/java/io/xpipe/core/source/DataTableConnection.java

@@ -0,0 +1,22 @@
+package io.xpipe.core.source;
+
+
+import io.xpipe.core.data.DataStructureNodeAcceptor;
+import io.xpipe.core.data.generic.ArrayNode;
+import io.xpipe.core.data.generic.TupleNode;
+import io.xpipe.core.data.type.TupleType;
+
+import java.io.OutputStream;
+
+public interface DataTableConnection extends DataSourceConnection {
+
+    TupleType determineDataType() throws Exception;
+
+    int determineRowCount() throws Exception;
+
+    void withLines(DataStructureNodeAcceptor<TupleNode> lineAcceptor) throws Exception;
+
+    ArrayNode readLines(int maxLines) throws Exception;
+
+    void forwardLines(OutputStream out, int maxLines) throws Exception;
+}

core/src/main/java/io/xpipe/core/source/DataTableSource.java

@@ -0,0 +1,15 @@
+package io.xpipe.core.source;
+
+import io.xpipe.core.store.DataStore;
+
+public abstract class DataTableSource<DS extends DataStore> implements DataSource<DS> {
+
+    public abstract DataTableWriteConnection openWriteConnection(DS store);
+
+    public abstract DataTableConnection openConnection(DS store);
+
+    @Override
+    public DataSourceType getType() {
+        return DataSourceType.TABLE;
+    }
+}

core/src/main/java/io/xpipe/core/source/DataTableWriteConnection.java

@@ -0,0 +1,12 @@
+package io.xpipe.core.source;
+
+import io.xpipe.core.data.DataStructureNodeAcceptor;
+import io.xpipe.core.data.generic.ArrayNode;
+import io.xpipe.core.data.generic.TupleNode;
+
+public interface DataTableWriteConnection extends DataSourceConnection {
+
+    DataStructureNodeAcceptor<TupleNode> writeLinesAcceptor();
+
+    void writeLines(ArrayNode lines) throws Exception;
+}

core/src/main/java/io/xpipe/core/store/DataStore.java

@@ -0,0 +1,18 @@
+package io.xpipe.core.store;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+import java.time.Instant;
+import java.util.Optional;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+public interface DataStore {
+
+    default Optional<String> determineDefaultName() {
+        return Optional.empty();
+    }
+
+    default Optional<Instant> getLastModified() {
+        return Optional.empty();
+    }
+}

core/src/main/java/io/xpipe/core/store/FileDataInput.java

@@ -0,0 +1,17 @@
+package io.xpipe.core.store;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public abstract class FileDataInput implements StreamDataStore {
+
+    public abstract String getName();
+
+    @JsonIgnore
+    public abstract boolean isLocal();
+
+    @JsonIgnore
+    public abstract LocalFileDataInput getLocal();
+
+    @JsonIgnore
+    public abstract RemoteFileDataInput getRemote();
+}

core/src/main/java/io/xpipe/core/store/InputStreamDataStore.java

@@ -0,0 +1,23 @@
+package io.xpipe.core.store;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public abstract class InputStreamDataStore implements StreamDataStore {
+
+    private final InputStream in;
+
+    public InputStreamDataStore(InputStream in) {
+        this.in = in;
+    }
+
+    @Override
+    public InputStream openInput() throws Exception {
+        return in;
+    }
+
+    @Override
+    public OutputStream openOutput() throws Exception {
+        throw new UnsupportedOperationException();
+    }
+}

core/src/main/java/io/xpipe/core/store/LocalFileDataInput.java

@@ -0,0 +1,73 @@
+package io.xpipe.core.store;
+
+import com.fasterxml.jackson.annotation.*;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.Optional;
+
+@JsonTypeName("local")
+public class LocalFileDataInput extends FileDataInput {
+
+    private final Path file;
+
+    @JsonCreator
+    public LocalFileDataInput(Path file) {
+        this.file = file;
+    }
+
+    @Override
+    public Optional<String> determineDefaultName() {
+        return Optional.of(FilenameUtils.getBaseName(file.toString()));
+    }
+
+    @Override
+    public Optional<Instant> getLastModified() {
+        try {
+            var l = Files.getLastModifiedTime(file);
+            return Optional.of(l.toInstant());
+        } catch (IOException e) {
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public String getName() {
+        return file.getFileName().toString();
+    }
+
+    @Override
+    @JsonIgnore
+    public boolean isLocal() {
+        return true;
+    }
+
+    @Override
+    public LocalFileDataInput getLocal() {
+        return this;
+    }
+
+    @Override
+    public RemoteFileDataInput getRemote() {
+        throw new UnsupportedOperationException();
+    }
+
+    public Path getFile() {
+        return file;
+    }
+
+    @Override
+    public InputStream openInput() throws Exception {
+        return Files.newInputStream(file);
+    }
+
+    @Override
+    public OutputStream openOutput() throws Exception {
+        return Files.newOutputStream(file);
+    }
+}

core/src/main/java/io/xpipe/core/store/RemoteFileDataInput.java

@@ -0,0 +1,49 @@
+package io.xpipe.core.store;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.time.Instant;
+import java.util.Optional;
+
+public class RemoteFileDataInput extends FileDataInput {
+
+    @Override
+    public Optional<String> determineDefaultName() {
+        return Optional.empty();
+    }
+
+    @Override
+    public Optional<Instant> getLastModified() {
+        return Optional.empty();
+    }
+
+    @Override
+    public String getName() {
+        return null;
+    }
+
+    @Override
+    public boolean isLocal() {
+        return false;
+    }
+
+    @Override
+    public LocalFileDataInput getLocal() {
+        return null;
+    }
+
+    @Override
+    public RemoteFileDataInput getRemote() {
+        return null;
+    }
+
+    @Override
+    public InputStream openInput() throws Exception {
+        return null;
+    }
+
+    @Override
+    public OutputStream openOutput() throws Exception {
+        return null;
+    }
+}

core/src/main/java/io/xpipe/core/store/StreamDataStore.java

@@ -0,0 +1,11 @@
+package io.xpipe.core.store;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public interface StreamDataStore extends DataStore {
+
+    InputStream openInput() throws Exception;
+
+    OutputStream openOutput() throws Exception;
+}

core/src/main/java/io/xpipe/core/util/CoreJacksonModule.java

@@ -0,0 +1,72 @@
+package io.xpipe.core.util;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import io.xpipe.core.data.type.ArrayType;
+import io.xpipe.core.data.type.TupleType;
+import io.xpipe.core.data.type.ValueType;
+import io.xpipe.core.store.LocalFileDataInput;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+
+public class CoreJacksonModule extends SimpleModule {
+
+    public static class CharsetSerializer extends JsonSerializer<Charset> {
+
+        @Override
+        public void serialize(Charset value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException, JsonProcessingException {
+            jgen.writeString(value.name());
+        }
+    }
+
+    public static class CharsetDeserializer extends JsonDeserializer<Charset> {
+
+        @Override
+        public Charset deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+            return Charset.forName(p.getValueAsString());
+        }
+    }
+
+    public static class LocalPathSerializer extends JsonSerializer<Path> {
+
+        @Override
+        public void serialize(Path value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException, JsonProcessingException {
+            jgen.writeString(value.toString());
+        }
+    }
+
+    public static class LocalPathDeserializer extends JsonDeserializer<Path> {
+
+        @Override
+        public Path deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+            return Path.of(p.getValueAsString());
+        }
+    }
+
+    @Override
+    public void setupModule(SetupContext context) {
+        context.registerSubtypes(
+                new NamedType(LocalFileDataInput.class),
+                new NamedType(ValueType.class),
+                new NamedType(TupleType.class),
+                new NamedType(ArrayType.class)
+        );
+
+        addSerializer(Charset.class, new CharsetSerializer());
+        addDeserializer(Charset.class, new CharsetDeserializer());
+
+        addSerializer(Path.class, new LocalPathSerializer());
+        addDeserializer(Path.class, new LocalPathDeserializer());
+    }
+}

core/src/main/java/io/xpipe/core/util/JacksonHelper.java

@@ -0,0 +1,48 @@
+package io.xpipe.core.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+public class JacksonHelper {
+
+    private static final ObjectMapper INSTANCE = new ObjectMapper();
+    private static boolean init = false;
+
+    public static synchronized void init(ModuleLayer layer) {
+        ObjectMapper objectMapper = INSTANCE;
+        objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+        objectMapper.registerModules(findModules(layer));
+        objectMapper.setVisibility(objectMapper.getSerializationConfig().getDefaultVisibilityChecker()
+                .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
+                .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
+                .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
+                .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)
+                .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE));
+        init = true;
+    }
+
+    private static List<Module> findModules(ModuleLayer layer)
+    {
+        ArrayList<Module> modules = new ArrayList<Module>();
+        ServiceLoader<Module> loader = ServiceLoader.load(layer, Module.class);
+        for (Module module : loader) {
+            modules.add(module);
+        }
+        return modules;
+    }
+
+    public static ObjectMapper newMapper() {
+        if (!init) {
+            throw new IllegalStateException("Not initialized");
+        }
+
+        return INSTANCE.copy();
+    }
+}

core/src/main/java/module-info.java

@@ -0,0 +1,28 @@
+import io.xpipe.core.util.CoreJacksonModule;
+
+module io.xpipe.core {
+    requires com.fasterxml.jackson.core;
+    requires com.fasterxml.jackson.databind;
+    requires com.fasterxml.jackson.module.paramnames;
+
+    exports io.xpipe.core.store;
+    exports io.xpipe.core.source;
+    exports io.xpipe.core.data.generic;
+    exports io.xpipe.core.data.type;
+
+    opens io.xpipe.core.store;
+    opens io.xpipe.core.source;
+    opens io.xpipe.core.data.type;
+    opens io.xpipe.core.data.generic;
+    exports io.xpipe.core.data.type.callback;
+    opens io.xpipe.core.data.type.callback;
+    exports io.xpipe.core.data;
+    opens io.xpipe.core.data;
+    exports io.xpipe.core.util;
+
+    uses com.fasterxml.jackson.databind.Module;
+    provides com.fasterxml.jackson.databind.Module with CoreJacksonModule;
+
+    requires org.apache.commons.lang;
+    requires org.apache.commons.io;
+}