diff --git a/.flake-modules/go-forwarder.nix b/.flake-modules/go-forwarder.nix index 647e54ee..34a38cf1 100644 --- a/.flake-modules/go-forwarder.nix +++ b/.flake-modules/go-forwarder.nix @@ -33,37 +33,39 @@ ... }: let - flakeRoot = nixpkgs-lib.getExe config.flake-root.package; - - # Build the networking/forwarder Go utility. - forwarder = pkgs.buildGoModule { - pname = "exo-forwarder"; - version = "0.1.0"; - src = "${flakeRoot}/networking/forwarder"; - - vendorHash = "sha256-BXIGg2QYqHDz2TNe8hLAGC6jVlffp9766H+WdkkuVgA="; - - # Only the main package at the repository root needs building. - subPackages = [ "." ]; - }; +# flakeRoot = nixpkgs-lib.getExe config.flake-root.package; +# +# # Build the networking/forwarder Go utility. +# forwarder = pkgs.buildGoModule { +# pname = "exo-forwarder"; +# version = "0.1.0"; +# src = "${flakeRoot}/networking/forwarder"; +# +# vendorHash = "sha256-BXIGg2QYqHDz2TNe8hLAGC6jVlffp9766H+WdkkuVgA="; +# +# # Only the main package at the repository root needs building. +# subPackages = [ "." ]; +# }; in { packages = { - inherit forwarder; +# inherit forwarder; }; apps = { - forwarder = { - type = "app"; - program = "${forwarder}/bin/forwarder"; - }; +# forwarder = { +# type = "app"; +# program = "${forwarder}/bin/forwarder"; +# }; }; make-shells.default = { # Go 1.24 compiler – align with go.mod packages = [ pkgs.go_1_24 ]; - shellHook = "export GOPATH=$FLAKE_ROOT/.go_cache"; + shellHook = '' + GOPATH="''$(${nixpkgs-lib.getExe config.flake-root.package})"/.go_cache + export GOPATH + ''; }; }; } - diff --git a/.flake-modules/just-flake.nix b/.flake-modules/just-flake.nix new file mode 100644 index 00000000..2208a58c --- /dev/null +++ b/.flake-modules/just-flake.nix @@ -0,0 +1,54 @@ +# Provides pretty banner & command index for this flake + +# Top-level parameters that are bound to the provider flake +# These are passed from `flake.nix` using importApply +{ + localSelf, + flake-parts-lib, + nixpkgs-lib, + just-flake, + ... +}: + +# These values would bind to the consumer flake when this flake module is imported: +{ + config, + self, + inputs, + getSystem, + moduleWithSystem, + withSystem, + ... +}: + +# The actual flake-parts module configuration +{ + imports = [ just-flake.flakeModule ]; + perSystem = + { + config, + self', + inputs', + pkgs, + system, + ... + }: + { + just-flake.features = { + # treefmt.enable = true; + # rust.enable = true; + # convco.enable = true; + # hello = { + # enable = true; + # justfile = '' + # hello: + # echo Hello World + # ''; + # }; + }; + + make-shells.default = { + inputsFrom = [ config.just-flake.outputs.devShell ]; + }; + }; +} diff --git a/.gitignore b/.gitignore index 3e73b059..200f8908 100644 --- a/.gitignore +++ b/.gitignore @@ -12,15 +12,15 @@ hosts_*.json # TODO figure out how to properly solve the issue with these target directories showing up networking/target/ networking/topology/target/ - -build/ -*.xcuserstate - rust/target/ rust/Cargo.lock +build/ +dist/ +*.xcuserstate + .DS_Store */.DS_Store # Says this symlink should be git-ignored https://github.com/juspay/just-flake -just-flake.just +just-flake.just \ No newline at end of file diff --git a/.idea/exo-v2.iml b/.idea/exo-v2.iml index e4d93c64..5357eaa9 100644 --- a/.idea/exo-v2.iml +++ b/.idea/exo-v2.iml @@ -5,15 +5,16 @@ + - - - - + + + + diff --git a/flake.lock b/flake.lock index 35e1853d..bc30d2b3 100644 --- a/flake.lock +++ b/flake.lock @@ -51,6 +51,21 @@ "type": "github" } }, + "just-flake": { + "locked": { + "lastModified": 1713316411, + "narHash": "sha256-NkJfU6H+6vgHkPtZ2ESbZ/h2wnsDQrZvB4vbdUIBx8Q=", + "owner": "juspay", + "repo": "just-flake", + "rev": "0e33952a4bcd16cd54ee3aba8111606c237d4526", + "type": "github" + }, + "original": { + "owner": "juspay", + "repo": "just-flake", + "type": "github" + } + }, "make-shell": { "inputs": { "flake-compat": "flake-compat" @@ -89,6 +104,7 @@ "inputs": { "flake-parts": "flake-parts", "flake-root": "flake-root", + "just-flake": "just-flake", "make-shell": "make-shell", "nixpkgs": "nixpkgs" } diff --git a/flake.nix b/flake.nix index 17253618..0098a869 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,9 @@ # 1. ${lib.getExe config.flake-root.package} # 2. $FLAKE_ROOT environment-varible flake-root.url = "github:srid/flake-root"; + + # Provides flake integration with [Just](https://just.systems/man/en/) + just-flake.url = "github:juspay/just-flake"; }; outputs = @@ -47,6 +50,7 @@ # instantiate all the flake modules, passing custom arguments to them as needed flakeModules = { flakeRoot = importApply' ./.flake-modules/flake-root.nix { inherit (inputs) flake-root; }; + justFlake = importApply' ./.flake-modules/just-flake.nix { inherit (inputs) just-flake; }; goForwarder = importApply' ./.flake-modules/go-forwarder.nix { }; }; in @@ -54,6 +58,7 @@ imports = [ inputs.make-shell.flakeModules.default flakeModules.flakeRoot + flakeModules.justFlake flakeModules.goForwarder ./.flake-modules/macmon.nix ]; diff --git a/justfile b/justfile index 53aaf70c..1b84e2eb 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,5 @@ +import 'just-flake.just' + default: @just --list @@ -45,3 +47,9 @@ run n="1" clean="false": if [ "{{clean}}" = "true" ]; then ./run.sh -rc; else ./run.sh -r; fi; \ done; \ fi + +# remote debugging auto-runner command: TODO: find better place to put this?? +# -> this pulls from upstream and wipes .exo folder, rebuilds & restarts +# -> TODO: maybe add a sync step for python deps ?? +autorun-master: + uv run scripts/watch-pull-restart.py --cmd "uv run exo-master" --restart-cmd "rm -rf ~/.exo && just build-forwarder" \ No newline at end of file diff --git a/networking/forwarder/go.mod b/networking/forwarder/go.mod index 47079c0f..51064579 100644 --- a/networking/forwarder/go.mod +++ b/networking/forwarder/go.mod @@ -1,12 +1,14 @@ module forwarder -go 1.24.3 +go 1.24.5 + +replace lib => ./lib replace forwarder/src => ./src require ( github.com/google/uuid v1.6.0 - github.com/libp2p/go-libp2p v0.42.1 + github.com/libp2p/go-libp2p v0.43.0 github.com/libp2p/go-libp2p-pubsub v0.14.2 github.com/mattn/go-sqlite3 v1.14.28 github.com/multiformats/go-multiaddr v0.16.0 @@ -22,10 +24,8 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect @@ -61,7 +61,6 @@ require ( github.com/multiformats/go-multistream v0.6.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.23.4 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect @@ -88,11 +87,11 @@ require ( github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.52.0 // indirect - github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/webtransport-go v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/wlynxg/anet v0.0.5 // indirect - go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect @@ -102,8 +101,8 @@ require ( golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.34.0 // indirect diff --git a/networking/forwarder/go.sum b/networking/forwarder/go.sum index 5ba5ce9e..2d13eb91 100644 --- a/networking/forwarder/go.sum +++ b/networking/forwarder/go.sum @@ -39,10 +39,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -62,8 +58,6 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -109,8 +103,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.42.1 h1:Rt8+5thie729NQk1gx1h/2t/+VIafWcqR1I+Kvw+UTg= -github.com/libp2p/go-libp2p v0.42.1/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= +github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvqXU= +github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-pubsub v0.14.2 h1:nT5lFHPQOFJcp9CW8hpKtvbpQNdl2udJuzLQWbgRum8= @@ -181,10 +175,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= -github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= -github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= -github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= @@ -233,8 +223,6 @@ github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= @@ -249,12 +237,12 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= -github.com/quic-go/quic-go v0.52.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= -github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= -github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= +github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= +github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -303,8 +291,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= @@ -385,8 +371,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -408,8 +394,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/networking/forwarder/lib/go.mod b/networking/forwarder/lib/go.mod new file mode 100644 index 00000000..9f10985e --- /dev/null +++ b/networking/forwarder/lib/go.mod @@ -0,0 +1,106 @@ +module lib + +go 1.24.5 + +require ( + github.com/ipfs/go-log/v2 v2.6.0 + github.com/stretchr/testify v1.10.0 + golang.org/x/sys v0.35.0 +) + +require ( + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/flynn/noise v1.1.0 // indirect + github.com/francoispqt/gojay v1.2.13 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect + github.com/ipfs/go-cid v0.5.0 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/koron/go-ssdp v0.0.6 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.2.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-netroute v0.2.2 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect + github.com/libp2p/go-yamux/v5 v5.0.1 // indirect + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/miekg/dns v1.1.66 // indirect + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect + github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.16.0 // indirect + github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect + github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.1 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-multistream v0.6.1 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pion/datachannel v1.5.10 // indirect + github.com/pion/dtls/v2 v2.2.12 // indirect + github.com/pion/dtls/v3 v3.0.6 // indirect + github.com/pion/ice/v4 v4.0.10 // indirect + github.com/pion/interceptor v0.1.40 // indirect + github.com/pion/logging v0.2.3 // indirect + github.com/pion/mdns/v2 v2.0.7 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.15 // indirect + github.com/pion/rtp v1.8.19 // indirect + github.com/pion/sctp v1.8.39 // indirect + github.com/pion/sdp/v3 v3.0.13 // indirect + github.com/pion/srtp/v3 v3.0.6 // indirect + github.com/pion/stun v0.6.1 // indirect + github.com/pion/stun/v3 v3.0.0 // indirect + github.com/pion/transport/v2 v2.2.10 // indirect + github.com/pion/transport/v3 v3.0.7 // indirect + github.com/pion/turn/v4 v4.0.2 // indirect + github.com/pion/webrtc/v4 v4.1.2 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.64.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/webtransport-go v0.9.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + go.uber.org/dig v1.19.0 // indirect + go.uber.org/fx v1.24.0 // indirect + go.uber.org/mock v0.5.2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + golang.org/x/mod v0.25.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/time v0.12.0 // indirect + golang.org/x/tools v0.34.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect + lukechampine.com/blake3 v1.4.1 // indirect +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/libp2p/go-libp2p v0.43.0 + github.com/pdgendt/cobs v1.1.0 + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sync v0.16.0 + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/networking/forwarder/lib/go.sum b/networking/forwarder/lib/go.sum new file mode 100644 index 00000000..b4e5ba17 --- /dev/null +++ b/networking/forwarder/lib/go.sum @@ -0,0 +1,443 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= +github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= +github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg= +github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= +github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= +github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= +github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvqXU= +github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc= +github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= +github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8= +github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= +github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= +github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= +github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.16.0 h1:oGWEVKioVQcdIOBlYM8BH1rZDWOGJSqr9/BKl6zQ4qc= +github.com/multiformats/go-multiaddr v0.16.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo= +github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= +github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pdgendt/cobs v1.1.0 h1:gGeI8VUIMCz5jAWoEi24UZv+vsQwiOSjoJuRY4jKnxg= +github.com/pdgendt/cobs v1.1.0/go.mod h1:AdxrOLm724a1y0E1RQn6+PtMjLUXgBM4FQJ9lm+/h3E= +github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= +github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= +github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E= +github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU= +github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= +github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= +github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= +github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= +github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= +github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= +github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= +github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= +github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= +github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= +github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= +github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= +github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= +github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= +github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= +github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= +github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= +github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= +github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= +github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= +github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= +github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= +github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= +github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= +go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= +go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= +lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/networking/forwarder/lib/ipc/flock_mutex.go b/networking/forwarder/lib/ipc/flock_mutex.go new file mode 100644 index 00000000..a15775ff --- /dev/null +++ b/networking/forwarder/lib/ipc/flock_mutex.go @@ -0,0 +1,208 @@ +//go:build unix + +package ipc + +import ( + "errors" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +var ( + ErrFileDescriptorAlreadyOpen = errors.New("file descriptor not open") + ErrFileDescriptorNotOpen = errors.New("file descriptor not open") + ErrLockAlreadyHeld = errors.New("lock already held") + ErrLockNotHeld = errors.New("lock not held") +) + +const ( + // open in read-write mode, creates file if it doesn't exist already, + // closes this file descriptor in any children processes (prevents FD leaking), + // truncates this file on opening (lock-files shouldn't hold content FOR NOW!!!) + // + // SEE: https://man7.org/linux/man-pages/man2/openat.2.html + flockMutexOpenFlags int = syscall.O_RDWR | syscall.O_CREAT | syscall.O_CLOEXEC | syscall.O_TRUNC + + // 0x644 mode flags -> user has read-write permissions, others have read permission only + // SEE: https://man7.org/linux/man-pages/man2/openat.2.html + flockMutexModeFlags uint32 = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IROTH + + // default poll-interval for spin-blocking lock + flockMutexPollInterval = 50 * time.Millisecond +) + +type LockType int + +const ( + ReadLock LockType = syscall.LOCK_SH + WriteLock LockType = syscall.LOCK_EX + LockMissing LockType = -1 +) + +type AcquireMode int + +const ( + OsBlocking AcquireMode = iota + SpinBlocking + NonBlocking +) + +type FlockMutex struct { + filePath string + fd int + lockHeld LockType +} + +func NewFlockMutex(filePath string) *FlockMutex { + return &FlockMutex{ + filePath: filePath, + fd: -1, + lockHeld: LockMissing, + } +} + +func (mu *FlockMutex) openFd() error { + if mu.fd != -1 { + return ErrFileDescriptorAlreadyOpen + } + // TODO: ensure_directory_exists(mu.filePath) + + // open file & TRY to change permissions to `modeFlags` flags + fd, err := unix.Open(mu.filePath, flockMutexOpenFlags, flockMutexModeFlags) + if err != nil { + return err + } else { + mu.fd = fd + _ = unix.Fchmod(fd, flockMutexModeFlags) // This locked is not owned by this UID + } + return nil +} + +func (mu *FlockMutex) closeFd() error { + if mu.fd == -1 { + return ErrFileDescriptorNotOpen + } + + if err := unix.Close(mu.fd); err != nil { + mu.fd = -1 + return err + } + + mu.fd = -1 + return nil +} + +func (mu *FlockMutex) acquire(lockType LockType, blocking bool) (bool, error) { + // enforce preconditions/sanity checks + if mu.fd == -1 { + return false, ErrFileDescriptorNotOpen + } + if mu.lockHeld != LockMissing { + return false, ErrLockAlreadyHeld + } + + // create flags for acquiring lock + var flags = int(lockType) + if !blocking { + flags |= syscall.LOCK_NB + } + + // continually try to acquire lock (since it may fail due to interrupts) + for { + if err := unix.Flock(mu.fd, flags); err != nil { + if errno, ok := err.(unix.Errno); ok { + // call interrupted by signal -> try again + if errno == unix.EINTR { + continue + } + + // file is locked & non-blocking is enabled -> return false to indicate + if errno == unix.EWOULDBLOCK { + return false, nil + } + } + + // unhandleable errors -> close FD & return error + _ = mu.closeFd() // TODO: how to merge Go errors ??? + return false, err + } + break + } + + // set lock-type held + mu.lockHeld = lockType + return true, nil +} + +func (mu *FlockMutex) release() error { + // enforce preconditions/sanity checks + if mu.fd == -1 { + return ErrFileDescriptorNotOpen + } + if mu.lockHeld == LockMissing { + return ErrLockNotHeld + } + + // continually try to release lock (since it may fail due to interrupts) + for { + if err := unix.Flock(mu.fd, syscall.LOCK_UN); err != nil { + if errno, ok := err.(unix.Errno); ok { + // call interrupted by signal -> try again + if errno == unix.EINTR { + continue + } + } + + // unhandleable errors -> close FD & return error + mu.lockHeld = LockMissing + _ = mu.closeFd() // TODO: how to merge Go errors ??? + return err + } + break + } + + mu.lockHeld = LockMissing + return nil +} + +func (mu *FlockMutex) Acquire(lockType LockType, acquireMode AcquireMode) (bool, error) { + // open file if missing + if mu.fd == -1 { + if err := mu.openFd(); err != nil { + return false, err + } + } + + // OS-blocking & non-blocking is direct passthrough to private function + switch acquireMode { + case OsBlocking: + return mu.acquire(lockType, true) + case NonBlocking: + return mu.acquire(lockType, false) + } + + // spin-blocking works by trying to acquire the lock in non-blocking mode, and retrying until success + for { + locked, err := mu.acquire(lockType, false) + if err != nil { + return false, err + } + if locked { + return true, err + } + time.Sleep(flockMutexPollInterval) + } +} + +func (mu *FlockMutex) Release(lockType LockType, acquireMode AcquireMode) error { + if err := mu.release(); err != nil { + _ = mu.closeFd() // TODO: how to merge Go errors ??? + return err + } + if err := mu.closeFd(); err != nil { + return err + } + return nil +} diff --git a/networking/forwarder/lib/ipc/flock_mutex_test.go b/networking/forwarder/lib/ipc/flock_mutex_test.go new file mode 100644 index 00000000..b0cb136f --- /dev/null +++ b/networking/forwarder/lib/ipc/flock_mutex_test.go @@ -0,0 +1,86 @@ +//go:build unix + +package ipc + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func check(t *testing.T, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func makeTempPath(t *testing.T, pattern string) string { + f, err := os.CreateTemp("", pattern) + check(t, err) + name := f.Name() + defer os.Remove(name) + return name +} + +func TestLockHeld(t *testing.T) { + path := makeTempPath(t, "testing_flock.lock") + defer os.Remove(path) + mu := NewFlockMutex(path) + + assert.Equal(t, LockMissing, mu.lockHeld) + + acquired, err := mu.Acquire(WriteLock, SpinBlocking) + check(t, err) + assert.True(t, acquired) + assert.Equal(t, WriteLock, mu.lockHeld) + check(t, mu.release()) + + assert.Equal(t, LockMissing, mu.lockHeld) + + acquired, err = mu.Acquire(ReadLock, SpinBlocking) + check(t, err) + assert.True(t, acquired) + assert.Equal(t, ReadLock, mu.lockHeld) + check(t, mu.release()) + + assert.Equal(t, LockMissing, mu.lockHeld) +} + +func TestNoReentrantLock(t *testing.T) { + path := makeTempPath(t, "testing_flock.lock") + defer os.Remove(path) + mu := NewFlockMutex(path) + + // no write-lock reentrancy + acquired, err := mu.Acquire(WriteLock, SpinBlocking) + check(t, err) + assert.True(t, acquired) + { + acquired, err = mu.Acquire(WriteLock, SpinBlocking) + assert.False(t, acquired) + assert.Equal(t, ErrLockAlreadyHeld, err) + } + { + acquired, err = mu.Acquire(ReadLock, SpinBlocking) + assert.False(t, acquired) + assert.Equal(t, ErrLockAlreadyHeld, err) + } + check(t, mu.release()) + + // no read-lock reentrancy + acquired, err = mu.Acquire(ReadLock, SpinBlocking) + check(t, err) + assert.True(t, acquired) + { + acquired, err = mu.Acquire(WriteLock, SpinBlocking) + assert.False(t, acquired) + assert.Equal(t, ErrLockAlreadyHeld, err) + } + { + acquired, err = mu.Acquire(ReadLock, SpinBlocking) + assert.False(t, acquired) + assert.Equal(t, ErrLockAlreadyHeld, err) + } + check(t, mu.release()) +} diff --git a/networking/forwarder/lib/ipc/pipe_duplex.go b/networking/forwarder/lib/ipc/pipe_duplex.go new file mode 100644 index 00000000..eeb0a396 --- /dev/null +++ b/networking/forwarder/lib/ipc/pipe_duplex.go @@ -0,0 +1,400 @@ +//go:build unix + +package ipc + +import ( + "bytes" + "context" + "errors" + "io/fs" + "lib" + "log" + "os" + "sync" + "syscall" + "time" + + "github.com/pdgendt/cobs" + "golang.org/x/sync/errgroup" + "golang.org/x/sys/unix" +) + +var ( + ErrInOutPipesAreSame = errors.New("the in-pipe and out-pipe are the same") + ErrExistingFileNotFifo = errors.New("the existing file is not a FIFO") +) + +const ( + pipeDuplexOpenReaderFlags = syscall.O_RDONLY | syscall.O_NONBLOCK + pipeDuplexOpenWriterFlags = syscall.O_WRONLY | syscall.O_NONBLOCK + pipeDuplexModeFlags = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IROTH + pipeDuplexPollInterval = 50 * time.Millisecond + pipeDuplex_PIPE_BUF = 4096 +) + +// Signal messages range from 1 to 255 & indicate control flow for the bytestream of the pipe. +type SignalMessage byte + +const ( + // DISCARD_PREVIOUS tells the receiver to discard previous partial work. + DiscardPrevious SignalMessage = 0x01 +) + +type OnMessage = func(msg []byte) error + +// Creates a named-pipe communication duplex. Creates a named-pipe communication duplex. +// The reader end is responsible for creating the pipe. +// +// The layers are: +// 1. Raw binary data over pipes +// 2. Variable-length binary packets with COBS +// 3. JSON-like values with Message Pack +type PipeDuplex struct { + inPath string + outPath string + + rawOutMu sync.Mutex + rawOut chan []byte + + ctx context.Context + cancel context.CancelFunc + errg *errgroup.Group +} + +func NewPipeDuplex(inPath, outPath string, onMessage OnMessage) (*PipeDuplex, error) { + // they must be different files + if inPath == outPath { + return nil, ErrInOutPipesAreSame + } + // pipes should only ever be created, and only by the reader (one-way operations) + if err := ensureFifoExists(inPath); err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + errg, ctx := errgroup.WithContext(ctx) + p := &PipeDuplex{ + inPath: inPath, + outPath: outPath, + + rawOut: make(chan []byte, 128), // TODO: decide on size of this w/ constant?? + + ctx: ctx, + cancel: cancel, + errg: errg, + } + // Reader + p.errg.Go(func() error { + return p.pipeBufferReader(onMessage) + }) + + // Writer + p.errg.Go(func() error { + return p.pipeBufferWriter() + }) + + return p, nil +} + +// Close stops all goroutines and waits for them to exit. +func (p *PipeDuplex) Close() error { + p.cancel() + + // this channel is exclusively written to via methods on this object handle, so it is its owner; + // owners must be the ones to close channels to avoid race conditions + defer func() { + // lock channel to avoid race conditions when closing + p.rawOutMu.Lock() + defer p.rawOutMu.Unlock() + + close(p.rawOut) + }() + + return p.errg.Wait() +} + +// SendMessage MessagePack-encodes a "value" and enqueues it to the writer. +func (p *PipeDuplex) SendMessage(msg []byte) error { + // lock channel to avoid race conditions when closing + p.rawOutMu.Lock() + defer p.rawOutMu.Unlock() + + // send message bytes over outRaw channel + select { + case p.rawOut <- msg: + // TODO: could this trigger a race condition if calling Close() immediately after SendMessage()??? + // should I lock p.rawOut w/ a mutex?? + return nil + case <-p.ctx.Done(): + return nil + } +} + +func (p *PipeDuplex) InPath() string { return p.inPath } +func (p *PipeDuplex) OutPath() string { return p.outPath } + +// ===== Private ===== + +func ensureFifoExists(path string) error { + // try to make a file if one doesn't exist already + // TODO: add equivalent of `ensure_parent_directory_exists(path)` here !!!!!! <- may cause bugs w/out it??? + if err := unix.Mkfifo(path, pipeDuplexModeFlags); err != nil { + if errno, ok := err.(unix.Errno); ok { + // misc error, do not handle + if errno != unix.EEXIST { + return err + } + + // ensure the file exists is FIFO + fi, err := os.Stat(path) + if err != nil { + return err // misc error, do not handle + } + if fi.Mode()&fs.ModeNamedPipe == 0 { + return ErrExistingFileNotFifo + } + return nil + } else { + return err // misc error, do not handle + } + } + return nil +} + +func (p *PipeDuplex) pipeBufferReader(onMessage OnMessage) error { + // open reader in nonblocking mode -> should not fail & immediately open; + // this marks when the writer process has "started" + fd, err := unix.Open(p.inPath, pipeDuplexOpenReaderFlags, pipeDuplexModeFlags) + if err != nil { + return err + } + defer unix.Close(fd) + + // continually pull from the pipe and interpret messages as such: + // - all messages are separated/framed by NULL bytes (zero) + // - messages with >=2 bytes are COBS-encoded messages, because + // the smallest COBS-encoded message is 2 bytes + // - 1-byte messages are therefore to be treated as control signals + var buf []byte // accumulation buffer + for { + select { // check for kill-signal + case <-p.ctx.Done(): + return nil + default: + } + + // read available data (and try again if nothing) + data := make([]byte, pipeDuplex_PIPE_BUF) + n, err := unix.Read(fd, data) + if err != nil { + errno, ok := err.(unix.Errno) + if !ok || errno != unix.EAGAIN { + return err + } + + // if there is a writer connected & the buffer is empty, this would block + // so we must consume this error gracefully and try again + time.Sleep(pipeDuplexPollInterval) + continue + } + if n == 0 { + time.Sleep(pipeDuplexPollInterval) + continue + } + + // extend buffer with new data + buf = append(buf, data[:n]...) + + // if there are no NULL bytes in the buffer, no new message has been formed + chunks := bytes.Split(buf, []byte{0x00}) + if len(chunks) == 1 { + continue + } + + // last chunk is always an unfinished message, so that becomes our new buffer; + // the rest should be decoded as either signals or COBS and put on queue + buf = chunks[len(chunks)-1] + for i := 0; i < len(chunks)-1; i++ { + chunk := chunks[i] + + // ignore empty messages (they mean nothing) + if len(chunk) == 0 { + continue + } + + // interpret 1-byte messages as signals (they indicate control-flow on messages) + if len(chunk) == 1 { + log.Printf("(reader): gotten control signal: %v", chunk[0]) + // TODO: do some kind of stuff here?? + continue + } + + // interpret >=2 byte messages as COBS-encoded data (decode them) + decoded, err := cobs.Decode(chunk) + if err != nil { + return err + } + + // call the callback to handle message + if err := onMessage(decoded); err != nil { + return err + } + } + } +} + +func (p *PipeDuplex) pipeBufferWriter() error { + log.Printf("(writer): started") + + // continually attempt to open FIFO for reading in nonblocking mode -> will error that: + // - ENOENT[2] No such file or directory: until a reader creates FIFO + // - ENXIO[6] No such device or address: until a reader opens FIFO + fd := -1 + for { + select { // check for kill-signal + case <-p.ctx.Done(): + return nil + default: + } + + tempFd, err := unix.Open(p.outPath, pipeDuplexOpenWriterFlags, pipeDuplexModeFlags) + if err != nil { + if errno, ok := err.(unix.Errno); ok { + // misc error, do not handle + if !(errno == unix.ENOENT || errno == unix.ENXIO) { + return err + } + + // try again if waiting for FIFO creation or reader-end opening + time.Sleep(pipeDuplexPollInterval) + continue + } else { + return err // misc error, do not handle + } + } + fd = tempFd + defer unix.Close(fd) + + // ensure the file exists is FIFO + mode, err := lib.FstatGetMode(fd) + if err != nil { + return err // misc error, do not handle + } + if mode&fs.ModeNamedPipe == 0 { + return ErrExistingFileNotFifo + } + + break // continue logic + } + + // read bytes from rawOut & write them to pipe + for { + select { + case buf, ok := <-p.rawOut: + if !ok { + return nil + } + if err := p.writeData(fd, buf); err != nil { + return err + } + case <-p.ctx.Done(): + return nil + } + + } +} + +func (p *PipeDuplex) writeData(fd int, buf []byte) error { + // COBS-encode the data & append NULL-byte to signify end-of-frame + buf, err := cobs.Encode(buf) + if err != nil { + return err + } + buf = append(buf, 0x00) + total := len(buf) + sent := 0 + + // begin transmission progress + for sent < total { + select { // check for kill-signal + case <-p.ctx.Done(): + return nil + default: + } + + // write & progress on happy path + written, err := unix.Write(fd, buf[sent:]) + if err == nil { + sent += written + continue + } + + // cast to OS error for propper handling + errno, ok := err.(unix.Errno) + if !ok { + return err // misc error, do not handle + } + + // non-blocking pipe is full, wait a bit and retry + if errno == syscall.EAGAIN { + time.Sleep(pipeDuplexPollInterval) + continue + } + + // reader disconnected -> handle failure-recovery by doing: + // 1. signal DISCARD_PREVIOUS to any reader + // 2. re-setting the progress & trying again + if errno == syscall.EPIPE { + if err := p.writeSignal(fd, DiscardPrevious); err != nil { + return err + } + sent = 0 + continue + } + + return err // misc error, do not handle + } + return nil +} + +func (p *PipeDuplex) writeSignal(fd int, sig SignalMessage) error { + signalMessageLength := 2 + + // Turn signal-byte into message by terminating with NULL-byte + buf := []byte{byte(sig), 0x00} + lib.Assert(len(buf) == signalMessageLength, "this must never NOT be the case") + + // attempt to write until successful + for { + select { // check for kill-signal + case <-p.ctx.Done(): + return nil + default: + } + + // small writes (e.g. 2 bytes) should be atomic as per Pipe semantics, + // meaning IF SUCCESSFUL: the number of bytes written MUST be exactly 2 + written, err := unix.Write(fd, buf) + if err == nil { + lib.Assert(written == signalMessageLength, "this must never NOT be the case") + break + } + + // cast to OS error for propper handling + errno, ok := err.(unix.Errno) + if !ok { + return err // misc error, do not handle + } + + // wait a bit and retry if: + // - non-blocking pipe is full + // - the pipe is broken because of reader disconnection + if errno == syscall.EAGAIN || errno == syscall.EPIPE { + time.Sleep(pipeDuplexPollInterval) + continue + } + + return err // misc error, do not handle + } + return nil +} diff --git a/networking/forwarder/lib/ipc/pipe_duplex_test.go b/networking/forwarder/lib/ipc/pipe_duplex_test.go new file mode 100644 index 00000000..7cd87b2d --- /dev/null +++ b/networking/forwarder/lib/ipc/pipe_duplex_test.go @@ -0,0 +1,85 @@ +//go:build unix + +package ipc + +import ( + "log" + "os" + "testing" + "time" +) + +func TestOneTwoThree(t *testing.T) { + // Avoid SIGPIPE killing the test if a writer outlives its reader. + // signal.Ignore(syscall.SIGPIPE) TODO: shoudn't sigpipe be handled by the error-code deep inside the duplex?? + + // Clean slate before/after. + onePath := "/tmp/one.pipe" + twoPath := "/tmp/two.pipe" + _ = os.Remove(onePath) + _ = os.Remove(twoPath) + defer os.Remove(onePath) + defer os.Remove(twoPath) + + owner, err := NewPipeDuplex( + onePath, // in + twoPath, // out + func(m []byte) error { log.Printf("wow, owner got: [%v]%v", len(m), m); return nil }, + ) + if err != nil { + t.Fatalf("owner New failed: %v", err) + } + + time.Sleep(1 * time.Second) + + guest1, err := NewPipeDuplex( + twoPath, // in + onePath, // out + func(m []byte) error { log.Printf("wow, guest1 got: [%v]%v", len(m), m); return nil }, + ) + if err != nil { + t.Fatalf("guest1 New failed: %v", err) + } + + if err := owner.SendMessage(make([]byte, 10)); err != nil { + t.Fatalf("owner SendMessage failed: %v", err) + } + + // batch send + if err := guest1.SendMessage(make([]byte, 200)); err != nil { + t.Fatalf("guest1 SendMessage failed: %v", err) + } + + time.Sleep(1 * time.Second) + + if err := guest1.Close(); err != nil { + t.Fatalf("guest1 Close failed: %v", err) + } + + if err := owner.SendMessage(make([]byte, 21)); err != nil { + t.Fatalf("owner SendMessage failed: %v", err) + } + + guest2, err := NewPipeDuplex( + twoPath, // in + onePath, // out + func(m []byte) error { log.Printf("wow, guest2 got: [%v]%v", len(m), m); return nil }, + ) + if err != nil { + t.Fatalf("guest2 New failed: %v", err) + } + + if err := guest2.SendMessage(make([]byte, 12)); err != nil { + t.Fatalf("guest2 SendMessage failed: %v", err) + } + + time.Sleep(1 * time.Second) + + if err := guest2.Close(); err != nil { + t.Fatalf("guest2 Close failed: %v", err) + } + if err := owner.Close(); err != nil { + t.Fatalf("owner Close failed: %v", err) + } + t.Fail() +} diff --git a/networking/forwarder/lib/libp2pext/dm/config.go b/networking/forwarder/lib/libp2pext/dm/config.go new file mode 100644 index 00000000..9fdc9d3e --- /dev/null +++ b/networking/forwarder/lib/libp2pext/dm/config.go @@ -0,0 +1,38 @@ +package dm + +import ( + "context" + + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" +) + +type Config struct { + Host host.Host + Protocol protocol.ID + MessageHandler MessageHandler + Logger *logging.ZapEventLogger +} + +type Option func(c *Config) error // TODO: add more options ?? + +func WithHandler(h MessageHandler) Option { + return func(c *Config) error { + c.MessageHandler = h + return nil + } +} +func WithHandlerFunction(onMessage func(ctx context.Context, from peer.ID, msg []byte) error) Option { + return func(c *Config) error { + c.MessageHandler = &MessageHandlerBundle{OnMessageF: onMessage} + return nil + } +} +func WithLogger(l *logging.ZapEventLogger) Option { + return func(c *Config) error { + c.Logger = l + return nil + } +} diff --git a/networking/forwarder/lib/libp2pext/dm/dm.go b/networking/forwarder/lib/libp2pext/dm/dm.go new file mode 100644 index 00000000..5cdba978 --- /dev/null +++ b/networking/forwarder/lib/libp2pext/dm/dm.go @@ -0,0 +1,57 @@ +package dm + +import ( + "context" + "errors" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" +) + +const ( + ServiceName = "libp2p.ext.dm/v1" + DmProtocol = protocol.ID("/dm/1.0.0") +) + +var ( + ErrMissingHandler = errors.New("the message handler is missing") +) + +type MessageHandler interface { + OnMessage(ctx context.Context, from peer.ID, msg []byte) error +} + +type MessageHandlerBundle struct { + OnMessageF func(ctx context.Context, from peer.ID, msg []byte) error +} + +func (m *MessageHandlerBundle) OnMessage(ctx context.Context, from peer.ID, msg []byte) error { + return m.OnMessageF(ctx, from, msg) +} + +type DirectMessenger interface { + Send(to peer.ID, msg []byte) error + Close() error +} + +func NewDirectMessenger(h host.Host, opts ...Option) (DirectMessenger, error) { + cfg := &Config{ + Host: h, + Protocol: DmProtocol, + Logger: logger, + } + + // apply all configs + for _, o := range opts { + if err := o(cfg); err != nil { + return nil, err + } + } + if cfg.MessageHandler == nil { + return nil, ErrMissingHandler + } + + // create DM from config + return newDirectMessenger(cfg) +} diff --git a/networking/forwarder/lib/libp2pext/dm/dm_test.go b/networking/forwarder/lib/libp2pext/dm/dm_test.go new file mode 100644 index 00000000..afa6cf02 --- /dev/null +++ b/networking/forwarder/lib/libp2pext/dm/dm_test.go @@ -0,0 +1,88 @@ +package dm + +import ( + "bytes" + "context" + "crypto/sha256" + "log" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic" + + "github.com/libp2p/go-libp2p" +) + +func genPriv(t *testing.T, seed [32]byte) crypto.PrivKey { + priv, _, err := crypto.GenerateEd25519Key(bytes.NewReader(seed[:])) + if err != nil { + t.Fatalf("failed generating key from seed %v: %v", seed, err) + } + return priv +} + +func createTestHost(t *testing.T, name string, opts ...Option) (host.Host, DirectMessenger) { + // generate key + seed := sha256.Sum256([]byte(name)) + id := genPriv(t, seed) + + // create host + h, err := libp2p.New( + libp2p.Identity(id), + libp2p.Transport(libp2pquic.NewTransport), + libp2p.ListenAddrStrings( + "/ip4/0.0.0.0/udp/0/quic-v1", + ), + ) + if err != nil { + t.Fatalf("failed creating test host '%v': %v", name, err) + } + + // configure direct messaging + dmOpts := []Option{WithHandler(&MessageHandlerBundle{ + OnMessageF: func(ctx context.Context, from peer.ID, msg []byte) error { + log.Printf("[%v]<-[%v]: [%v]%v", name, from, len(msg), msg) + return nil + }, + })} + dmOpts = append(dmOpts, opts...) + dm, err := NewDirectMessenger(h, dmOpts...) + if err != nil { + t.Fatalf("failed creating test DM manager for host '%v': %v", name, err) + } + + return h, dm +} + +func createConnection(t *testing.T, p1, p2 host.Host) { + ctx := context.Background() + if err := p1.Connect(ctx, p2.Peerstore().PeerInfo(p2.ID())); err != nil { + t.Fatalf("failed connecting '%v' to '%v': %v", p1.ID(), p2.ID(), err) + } +} + +func TestJsonEncoder(t *testing.T) { + peer1, dm1 := createTestHost(t, "peer 1") + defer dm1.Close() + defer peer1.Close() + + peer2, dm2 := createTestHost(t, "peer 2") + defer dm2.Close() + defer peer2.Close() + + createConnection(t, peer1, peer2) + + if err := dm1.Send(peer2.ID(), make([]byte, 10)); err != nil { + t.Fatalf("dm1 Send failed: %v", err) + } + + // big send + if err := dm2.Send(peer1.ID(), make([]byte, 10_000)); err != nil { + t.Fatalf("dm2 Send failed: %v", err) + } + time.Sleep(500 * time.Millisecond) + t.Fail() +} diff --git a/networking/forwarder/lib/libp2pext/dm/internal.go b/networking/forwarder/lib/libp2pext/dm/internal.go new file mode 100644 index 00000000..24b7dff3 --- /dev/null +++ b/networking/forwarder/lib/libp2pext/dm/internal.go @@ -0,0 +1,151 @@ +package dm + +import ( + "context" + "encoding/binary" + "io" + "lib" + "sync" + + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/proto" +) + +const ( + uint64NumBytes = 8 +) + +var ( + logger = logging.Logger(ServiceName) +) + +type directMessenger struct { + ctx context.Context + cancel func() + + h host.Host + pid protocol.ID + handler MessageHandler + log *logging.ZapEventLogger + + scope network.ResourceScopeSpan + notifiee network.Notifiee + + mx sync.Mutex + closed bool +} + +func newDirectMessenger(cfg *Config) (*directMessenger, error) { + ctx, cancel := context.WithCancel(context.Background()) + dm := &directMessenger{ + ctx: ctx, + cancel: cancel, + + h: cfg.Host, + pid: cfg.Protocol, + handler: cfg.MessageHandler, + log: cfg.Logger, + } + + // get a scope for memory reservations at service level + err := dm.h.Network().ResourceManager().ViewService(ServiceName, + func(s network.ServiceScope) error { + var err error + dm.scope, err = s.BeginSpan() + return err + }) + if err != nil { + return nil, err + } + + dm.h.SetStreamHandler(dm.pid, dm.handleStream) + dm.notifiee = &network.NotifyBundle{} // TODO: add handler funcions in the future if so needed?? + dm.h.Network().Notify(dm.notifiee) + + return dm, nil +} + +func (dm *directMessenger) Close() error { + dm.mx.Lock() + if !dm.closed { + dm.closed = true + dm.mx.Unlock() + + dm.h.RemoveStreamHandler(proto.ProtoIDv2Hop) + dm.h.Network().StopNotify(dm.notifiee) + defer dm.scope.Done() + dm.cancel() + return nil + } + dm.mx.Unlock() + return nil +} + +func (dm *directMessenger) Send(p peer.ID, msg []byte) error { + dm.log.Infof("outgoing DM stream to: %s", p) + + // create new stream + s, err := dm.h.NewStream(dm.ctx, p, dm.pid) + if err != nil { + return err + } + defer s.Close() + + // grab length if byte-buffer and encode it as big-endian + mLen := len(msg) + buf := make([]byte, uint64NumBytes, uint64NumBytes+mLen) // allocate enough capacity + binary.BigEndian.PutUint64(buf, uint64(mLen)) + buf = append(buf, msg...) + lib.Assert(len(buf) == uint64NumBytes+mLen, "literally what????") + + // write to stream & handle any potential errors + if _, err := s.Write(buf); err != nil { + dm.log.Debugf("error writing message to DM service stream: %s", err) + s.Reset() + return err + } + + _ = s.CloseWrite() // signal EOF to caller if half-close is supported + return nil +} + +func (dm *directMessenger) handleStream(s network.Stream) { + dm.log.Infof("incoming DM stream from: %s", s.Conn().RemotePeer()) + + defer s.Close() + + // attach scope to this service (for scoped capacity allocation reasons) + if err := s.Scope().SetService(ServiceName); err != nil { + dm.log.Debugf("error attaching stream to DM service: %s", err) + s.Reset() + return + } + + // read big-endian length bytes & decode + buf := make([]byte, uint64NumBytes) + if _, err := io.ReadFull(s, buf); err != nil { + dm.log.Debugf("error reading message length from DM service stream: %s", err) + s.Reset() + return + } + mLen := binary.BigEndian.Uint64(buf) + + // read rest of message & call OnMessage callback + buf = make([]byte, mLen) + if _, err := io.ReadFull(s, buf); err != nil { + dm.log.Debugf("error reading message body from DM service stream: %s", err) + s.Reset() + return + } + if err := dm.handler.OnMessage(dm.ctx, s.Conn().RemotePeer(), buf); err != nil { + dm.log.Debugf("error handling incoming message from DM service stream: %s", err) + s.Reset() + return + } + + _ = s.CloseWrite() // signal EOF to caller if half-close is supported +} diff --git a/networking/forwarder/lib/util.go b/networking/forwarder/lib/util.go new file mode 100644 index 00000000..879b9ba3 --- /dev/null +++ b/networking/forwarder/lib/util.go @@ -0,0 +1,52 @@ +package lib + +import ( + "log" + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +func Assert(b bool, msg string) { + if !b { + log.Panic(msg) + } +} + +func FstatGetMode(fd int) (os.FileMode, error) { + // perform fstat syscall + var sys unix.Stat_t = unix.Stat_t{} + if err := unix.Fstat(fd, &sys); err != nil { + return 0, err + } + + // reconstruct FileMode from sys-struct; SEE: https://github.com/golang/go/blob/5a56d8848b4ffb79c5ccc11ec6fa01823a91aaf8/src/os/stat_linux.go#L17 + mode := os.FileMode(sys.Mode & 0777) + switch sys.Mode & syscall.S_IFMT { + case syscall.S_IFBLK: + mode |= os.ModeDevice + case syscall.S_IFCHR: + mode |= os.ModeDevice | os.ModeCharDevice + case syscall.S_IFDIR: + mode |= os.ModeDir + case syscall.S_IFIFO: + mode |= os.ModeNamedPipe + case syscall.S_IFLNK: + mode |= os.ModeSymlink + case syscall.S_IFREG: + // nothing to do + case syscall.S_IFSOCK: + mode |= os.ModeSocket + } + if sys.Mode&syscall.S_ISGID != 0 { + mode |= os.ModeSetgid + } + if sys.Mode&syscall.S_ISUID != 0 { + mode |= os.ModeSetuid + } + if sys.Mode&syscall.S_ISVTX != 0 { + mode |= os.ModeSticky + } + return mode, nil +} diff --git a/pyproject.toml b/pyproject.toml index 20f8b5b6..d43868ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "mlx-lm @ https://github.com/ml-explore/mlx-lm.git", "psutil>=7.0.0", "transformers>=4.55.2", + "cobs>=1.2.2", ] [project.scripts] diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index 8d10af64..54bf6702 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -5,10 +5,13 @@ description = "Scripts for the Exo project" readme = "README.md" requires-python = ">=3.13" dependencies = [ - "shared", "huggingface_hub>=0.33.4", + "exo" ] [build-system] requires = ["uv_build>=0.8.9,<0.9.0"] build-backend = "uv_build" + +[tool.uv.sources] +exo = { workspace = true } \ No newline at end of file diff --git a/scripts/src/exo_scripts/read_events.py b/scripts/src/exo_scripts/read_events.py index f8da5679..68fc9398 100644 --- a/scripts/src/exo_scripts/read_events.py +++ b/scripts/src/exo_scripts/read_events.py @@ -1,9 +1,10 @@ +# pyright: reportAny=false + import asyncio import curses import time import json import argparse -import textwrap import sys from logging import Logger from typing import List, Optional, Any, Sequence, Tuple @@ -27,13 +28,17 @@ WORKER_EVENT_TYPES = { 'RunnerStatusUpdated', 'RunnerDeleted' } + async def init_db() -> None: global event_log_manager event_log_manager = EventLogManager(EventLogConfig(), logger) await event_log_manager.initialize() + async def get_events_since(since: int) -> Sequence[EventFromEventLog[Event]]: - return await event_log_manager.global_events.get_events_since(since) # type: ignore[attr-defined, return-value] + assert event_log_manager is not None + return await event_log_manager.global_events.get_events_since(since) + async def load_all_events() -> List[EventFromEventLog[Event]]: events: List[EventFromEventLog[Event]] = [] @@ -46,6 +51,7 @@ async def load_all_events() -> List[EventFromEventLog[Event]]: since += len(new_events) return events + def compute_states(events: List[EventFromEventLog[Event]]) -> List[State]: states: List[State] = [State()] state = states[0] @@ -54,12 +60,15 @@ def compute_states(events: List[EventFromEventLog[Event]]) -> List[State]: states.append(state) return states + def print_event(event: EventFromEventLog[Event]) -> None: event_type_name = type(event.event).__name__ event_type = event_type_name.replace('_', ' ').title() - attributes = ', '.join(f"{key}={value!r}" for key, value in vars(event.event).items()) + attributes = ', '.join(f"{key}={value!r}" for key, + value in vars(event.event).items()) print(f"[{event.idx_in_log}] {event_type}: {attributes}") + async def non_tui_mode() -> None: await init_db() events = await load_all_events() @@ -67,7 +76,8 @@ async def non_tui_mode() -> None: final_state = states[-1] if worker_mode: - filtered_events = [e for e in events if type(e.event).__name__ in WORKER_EVENT_TYPES] + filtered_events = [e for e in events if type( + e.event).__name__ in WORKER_EVENT_TYPES] events = filtered_events # Recompute states? But states are cumulative, so perhaps just print filtered events and full state, or filter state too. state_dict = json.loads(final_state.model_dump_json()) @@ -88,7 +98,9 @@ async def non_tui_mode() -> None: for event in events: print_event(event) -async def update_events(wrapped_events: List[EventFromEventLog[Event]], states: List[State], filtered_indices: Optional[List[int]] = None) -> bool: + +async def update_events(wrapped_events: List[EventFromEventLog[Event]], states: List[State], + filtered_indices: Optional[List[int]] = None) -> bool: last_since = len(wrapped_events) new_wrapped = await get_events_since(last_since) if new_wrapped: @@ -105,6 +117,7 @@ async def update_events(wrapped_events: List[EventFromEventLog[Event]], states: return True return False + def draw_state(win: Any, state: State, height: int, width: int, worker_mode: bool, state_scroll: int) -> int: win.clear() state_dict = json.loads(state.model_dump_json()) @@ -142,11 +155,13 @@ def draw_state(win: Any, state: State, height: int, width: int, worker_mode: boo value_str = stripped[end_key + 3:] if value_str.startswith('"'): color = 2 - elif value_str.replace('.', '', 1).isdigit() or (value_str.startswith('-') and value_str[1:].replace('.', '', 1).isdigit()): + elif value_str.replace('.', '', 1).isdigit() or ( + value_str.startswith('-') and value_str[1:].replace('.', '', 1).isdigit()): color = 4 elif value_str in ['true', 'false', 'null']: color = 5 - elif value_str.startswith('{') or value_str.startswith('[') or value_str.startswith('}') or value_str.startswith(']'): + elif value_str.startswith('{') or value_str.startswith('[') or value_str.startswith( + '}') or value_str.startswith(']'): color = 0 else: color = 0 @@ -158,6 +173,7 @@ def draw_state(win: Any, state: State, height: int, width: int, worker_mode: boo win.refresh() return current_scroll + def get_event_pairs(event: EventFromEventLog[Event]) -> List[Tuple[str, int]]: pairs: List[Tuple[str, int]] = [] idx_str = f"[{event.idx_in_log}] " @@ -186,6 +202,7 @@ def get_event_pairs(event: EventFromEventLog[Event]) -> List[Tuple[str, int]]: pairs.append((v_str, color)) return pairs + def calculate_event_lines(pairs: List[Tuple[str, int]], win_width: int, subsequent_indent: int) -> int: lines = 1 x = 0 @@ -201,7 +218,9 @@ def calculate_event_lines(pairs: List[Tuple[str, int]], win_width: int, subseque x = subsequent_indent return lines -def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold: bool, win_width: int, subsequent_indent: int) -> int: + +def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold: bool, win_width: int, + subsequent_indent: int) -> int: y = start_y x = 0 for text, color in pairs: @@ -226,6 +245,7 @@ def render_event(win: Any, start_y: int, pairs: List[Tuple[str, int]], is_bold: y += 1 return y + def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_events: int, height: int) -> None: win.clear() if len(events_list) == 0: @@ -236,7 +256,8 @@ def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_e current_event = events_list[current_events] current_pairs = get_event_pairs(current_event) subsequent_indent = len(f"[{current_event.idx_in_log}] ") - lines_current = calculate_event_lines(current_pairs, win_width, subsequent_indent) + lines_current = calculate_event_lines( + current_pairs, win_width, subsequent_indent) if lines_current > height: render_event(win, 0, current_pairs, True, win_width, subsequent_indent) win.refresh() @@ -313,12 +334,15 @@ def draw_events(win: Any, events_list: List[EventFromEventLog[Event]], current_e win.refresh() + def draw_status(win: Any, realtime: bool, current: int, total_events: int) -> None: win.clear() mode = "Realtime" if realtime else "Timetravel" - win.addstr(0, 0, f"Mode: {mode} | Current event: {current} / {total_events} | Arrows: navigate events, [/]: scroll state, g: goto, r: toggle realtime, q: quit") + win.addstr(0, 0, + f"Mode: {mode} | Current event: {current} / {total_events} | Arrows: navigate events, [/]: scroll state, g: goto, r: toggle realtime, q: quit") win.refresh() + def get_input(stdscr: Any, prompt: str) -> str: curses.echo() stdscr.addstr(0, 0, prompt) @@ -327,6 +351,7 @@ def get_input(stdscr: Any, prompt: str) -> str: curses.noecho() return input_str + def get_key(win: Any) -> Any: ch = win.getch() if ch == -1: @@ -369,6 +394,7 @@ def get_key(win: Any) -> Any: return 'CTRL_DOWN' return ch + def tui(stdscr: Any) -> None: curses.start_color() curses.init_pair(1, curses.COLOR_BLUE, curses.COLOR_BLACK) @@ -390,8 +416,10 @@ def tui(stdscr: Any) -> None: current_filtered: int = -1 current: int = -1 if worker_mode: - filtered_indices = [i for i in range(len(wrapped_events)) if type(wrapped_events[i].event).__name__ in WORKER_EVENT_TYPES] - current_filtered = len(filtered_indices) - 1 if filtered_indices else -1 + filtered_indices = [i for i in range(len(wrapped_events)) if + type(wrapped_events[i].event).__name__ in WORKER_EVENT_TYPES] + current_filtered = len(filtered_indices) - \ + 1 if filtered_indices else -1 else: current = len(wrapped_events) - 1 if wrapped_events else -1 @@ -407,7 +435,8 @@ def tui(stdscr: Any) -> None: pane_width = width // 2 state_win = curses.newwin(pane_height, pane_width, 0, 0) - events_win = curses.newwin(pane_height, width - pane_width, 0, pane_width) + events_win = curses.newwin( + pane_height, width - pane_width, 0, pane_width) status_win = curses.newwin(status_height, width, pane_height, 0) if worker_mode: @@ -421,10 +450,12 @@ def tui(stdscr: Any) -> None: current_events = current state_idx = current_original + 1 if current_original >= 0 else 0 - state_scroll = draw_state(state_win, states[state_idx], pane_height, pane_width, worker_mode, state_scroll) + state_scroll = draw_state( + state_win, states[state_idx], pane_height, pane_width, worker_mode, state_scroll) draw_events(events_win, events_list, current_events, pane_height) total_events = len(wrapped_events) - 1 if wrapped_events else -1 - draw_status(status_win, realtime, current_original if worker_mode else current, total_events) + draw_status(status_win, realtime, + current_original if worker_mode else current, total_events) key = get_key(stdscr) if key != -1: @@ -439,13 +470,16 @@ def tui(stdscr: Any) -> None: else: current = max(0, current - 5) elif key == curses.KEY_DOWN: - if worker_mode and current_filtered < len(filtered_indices) - 1: # type: ignore[arg-type] + assert filtered_indices is not None + if worker_mode and current_filtered < len(filtered_indices) - 1: current_filtered += 1 elif not worker_mode and current < len(wrapped_events) - 1: current += 1 elif key == 'CTRL_DOWN': + assert filtered_indices is not None if worker_mode: - current_filtered = min(len(filtered_indices) - 1, current_filtered + 5) # type: ignore[arg-type] + current_filtered = min( + len(filtered_indices) - 1, current_filtered + 5) else: current = min(len(wrapped_events) - 1, current + 5) elif key == ord('['): @@ -457,10 +491,13 @@ def tui(stdscr: Any) -> None: elif key == ord('r'): realtime = not realtime if realtime: + assert filtered_indices is not None if worker_mode: - current_filtered = len(filtered_indices) - 1 if filtered_indices else -1 # type: ignore[arg-type] + current_filtered = len( + filtered_indices) - 1 if filtered_indices else -1 else: - current = len(wrapped_events) - 1 if wrapped_events else -1 + current = len(wrapped_events) - \ + 1 if wrapped_events else -1 state_scroll = 0 elif key == ord('g'): stdscr.timeout(-1) # block for input @@ -487,18 +524,23 @@ def tui(stdscr: Any) -> None: status_win.refresh() if realtime and time.time() - last_update > update_interval: - updated = asyncio.run(update_events(wrapped_events, states, filtered_indices if worker_mode else None)) + updated = asyncio.run(update_events( + wrapped_events, states, filtered_indices if worker_mode else None)) if updated: + assert filtered_indices is not None if worker_mode: - current_filtered = len(filtered_indices) - 1 # type: ignore[arg-type] + current_filtered = len(filtered_indices) - 1 else: current = len(wrapped_events) - 1 state_scroll = 0 last_update = time.time() + if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Read and display events from the event log') - parser.add_argument('--worker', action='store_true', help='Only show worker-related events (task, streaming, instance, runner status)') + parser = argparse.ArgumentParser( + description='Read and display events from the event log') + parser.add_argument('--worker', action='store_true', + help='Only show worker-related events (task, streaming, instance, runner status)') args = parser.parse_args() worker_mode = args.worker @@ -513,4 +555,4 @@ if __name__ == "__main__": print("Error: Could not find terminal. Falling back to non-TUI mode.") asyncio.run(non_tui_mode()) else: - raise \ No newline at end of file + raise diff --git a/scripts/watch-pull-restart.py b/scripts/watch-pull-restart.py new file mode 100755 index 00000000..aad5c0b2 --- /dev/null +++ b/scripts/watch-pull-restart.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 + +""" +watch-pull-restart.py — Unix-only + +Runs a command, periodically checks git upstream, pulls if upstream is ahead, +and gracefully restarts the command. Watcher logs go to STDERR; your app's +output goes straight to the console (STDOUT/STDERR). + +Assumptions: + - current branch tracks an upstream (i.e., @{u} exists) + - pulls must be fast-forward (remote-ahead workflow) + +Arguments: + - cmd: Command to run/manage (e.g. './run.sh' or 'python -m app'). + - restart-cmd: Optional hook to run after a successful pull (e.g., systemctl restart). + - sleep-secs: Poll interval while up-to-date. + - grace-secs: Seconds to wait after SIGTERM before SIGKILL. + - debounce-secs: Coalesce multiple pulls before restart. + +Usage: + ./watch-pull-restart.py --cmd "./run.sh" --sleep-secs 1 + ./watch-pull-restart.py --cmd "python -m app" --restart-cmd "systemctl --user restart myapp" + ./watch-pull-restart.py --restart-cmd "systemctl --user restart myapp" # no managed child; only trigger hook +""" +import argparse +import os +import signal +import subprocess +import sys +import time +from types import FrameType +from typing import Optional + + +# ---------- logging helpers (to STDERR) ---------- +def log(msg: str): + sys.stderr.write(msg.rstrip() + "\n") + sys.stderr.flush() + + +def sep(title: str = ""): + """Big visual separator for state transitions (to STDERR).""" + sys.stderr.write("\n\n") + if title: + sys.stderr.write(f"===== [watch] {title} =====\n") + else: + sys.stderr.write("===== [watch] =====\n") + sys.stderr.flush() + + +def run_capture(cmd: str, check: bool = True) -> subprocess.CompletedProcess[str]: + """Run and capture output; for git plumbing.""" + return subprocess.run( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=check, + ) + + +# ---------- shell helpers ---------- +def is_up_to_date() -> bool: + subprocess.run("git fetch --quiet", + shell=True) # Quiet fetch; ignore network errors (we'll just try again next tick) + try: + current = run_capture("git rev-parse HEAD", check=True).stdout.strip() + upstream = run_capture("git rev-parse @{u}", check=True).stdout.strip() + return current == upstream + except subprocess.CalledProcessError: + return True # No upstream or other git error; treat as up-to-date to avoid thrash + + +def pull_ff_only() -> bool: + """Returns True if pull applied changes, False if already up-to-date.""" + try: + cp = run_capture("git pull --ff-only --no-rebase", check=True) + return "Already up to date" not in cp.stdout and cp.returncode == 0 # Git prints "Already up to date." on no-op; cheap heuristic + except subprocess.CalledProcessError as e: + log("[watch] git pull failed:") + if e.stdout: # pyright: ignore[reportAny] + log(e.stdout) # pyright: ignore[reportAny] + if e.stderr: # pyright: ignore[reportAny] + log(e.stderr) # pyright: ignore[reportAny] + return False + + +# ---------- managed processes ---------- +class ManagedProc: + def __init__(self, cmd: Optional[str], grace_secs: float): + self.cmd = cmd + self.grace = grace_secs + self.child: Optional[subprocess.Popen[bytes]] = None + + def start(self): + if not self.cmd: + return + if self.child and self.child.poll() is None: + return + sep("starting main cmd") + log(f"[watch] starting: {self.cmd}") + # New process group so we can signal the entire tree (shell + children) + self.child = subprocess.Popen( + self.cmd, + shell=True, # allow shell features in --cmd + stdout=None, # inherit parent's stdout (your app prints normally) + stderr=None, # inherit parent's stderr + stdin=None, + preexec_fn=os.setsid, # create new session (PGID == child PID) + ) + + def stop_gracefully(self): + if not self.child: + return + if self.child.poll() is not None: + self.child = None + return + + sep("stopping main cmd (SIGTERM)") + try: + os.killpg(self.child.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + deadline = time.time() + self.grace + while time.time() < deadline: + if self.child.poll() is not None: + self.child = None + return + time.sleep(0.1) + + sep("main cmd unresponsive; SIGKILL") + try: + os.killpg(self.child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.child = None + + def forward_signal(self, sig: int): + if not self.child or self.child.poll() is not None: + return + try: + os.killpg(self.child.pid, sig) + except ProcessLookupError: + pass + + +class OneShotHook: + """ + One-shot hook command (e.g., systemctl restart). + Runs to completion with inherited stdio so its output is visible. + """ + + def __init__(self, cmd: Optional[str], grace_secs: float): + self.cmd = cmd + self.grace = grace_secs + self.child: Optional[subprocess.Popen[bytes]] = None + + def run(self) -> int: + if not self.cmd: + return 0 + sep("running restart hook") + log(f"[watch] hook: {self.cmd}") + self.child = subprocess.Popen( + self.cmd, + shell=True, + stdout=None, # inherit stdio + stderr=None, + stdin=None, + preexec_fn=os.setsid, + ) + # Wait with grace/kill if needed (rare for hooks, but symmetric) + deadline = time.time() + self.grace + while True: + rc = self.child.poll() + if rc is not None: + self.child = None + return rc + if time.time() > deadline: + sep("hook exceeded grace; SIGKILL") + try: + os.killpg(self.child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.child = None + return 137 # killed + time.sleep(0.1) + + def forward_signal(self, sig: int): + if not self.child or self.child.poll() is not None: + return + try: + os.killpg(self.child.pid, sig) + except ProcessLookupError: + pass + + +# ---------- main loop ---------- +def main(): + # CMD commands + ap = argparse.ArgumentParser(description="Auto-pull & restart on upstream changes (Unix).") + ap.add_argument("--cmd", help="Command to run/manage (e.g. './run.sh' or 'python -m app').") + ap.add_argument("--restart-cmd", help="Optional hook to run after a successful pull (e.g., systemctl restart).") + ap.add_argument("--sleep-secs", type=float, default=0.5, help="Poll interval while up-to-date.") + ap.add_argument("--grace-secs", type=float, default=5.0, help="Seconds to wait after SIGTERM before SIGKILL.") + ap.add_argument("--debounce-secs", type=float, default=0.5, help="Coalesce multiple pulls before restart.") + args = ap.parse_args() + + # get CMD command values + cmd = args.cmd # pyright: ignore[reportAny] + assert cmd is None or isinstance(cmd, str) + restart_cmd = args.restart_cmd # pyright: ignore[reportAny] + assert cmd is None or isinstance(restart_cmd, str) + sleep_secs = args.sleep_secs # pyright: ignore[reportAny] + assert sleep_secs is not None and isinstance(sleep_secs, float) + grace_secs = args.grace_secs # pyright: ignore[reportAny] + assert sleep_secs is not None and isinstance(grace_secs, float) + debounce_secs = args.debounce_secs # pyright: ignore[reportAny] + assert sleep_secs is not None and isinstance(debounce_secs, float) + + # start managed proc + proc = ManagedProc(cmd, grace_secs) + hook = OneShotHook(restart_cmd, grace_secs) + + # signal handling for graceful exit + exiting = {"flag": False} + + def _handle(sig_num: int, _frame: Optional[FrameType]): + sep(f"received signal {sig_num}; exiting") + exiting["flag"] = True + proc.forward_signal(sig_num) + hook.forward_signal(sig_num) + + signal.signal(signal.SIGINT, _handle) + signal.signal(signal.SIGTERM, _handle) + + # Initial start (if managing a process) + proc.start() + + pending_restart = False + last_change = 0.0 + while not exiting["flag"]: + try: + if not is_up_to_date(): + sep("upstream ahead; pulling") + changed = pull_ff_only() + if changed: + last_change = time.time() + pending_restart = True + + # handle debounce window + if pending_restart and (time.time() - last_change) >= debounce_secs: + # Optional hook first + if restart_cmd: + rc = hook.run() + if rc != 0: + sep(f"hook exited with {rc}") + # Then bounce managed process + if cmd: + proc.stop_gracefully() + proc.start() + pending_restart = False + sep("restart cycle complete") + + # keep the child alive if it crashed without a pull + if cmd and (proc.child is None or proc.child.poll() is not None): + sep("main cmd exited; restarting") + proc.start() + + time.sleep(sleep_secs) + except Exception as e: + sep("loop error") + log(f"[watch] {e}") + time.sleep(2.0) + + # graceful shutdown on exit + proc.stop_gracefully() + sep("bye") + + +if __name__ == "__main__": + main() diff --git a/src/exo/engines/mlx/auto_parallel.py b/src/exo/engines/mlx/auto_parallel.py index 2e2589fa..383cb8c2 100644 --- a/src/exo/engines/mlx/auto_parallel.py +++ b/src/exo/engines/mlx/auto_parallel.py @@ -1,7 +1,7 @@ from typing import Protocol, cast, override import mlx.core as mx -import mlx.nn as nn +import mlx.nn as nn # pyright: ignore[reportMissingTypeStubs] from exo.shared.types.worker.shards import PipelineShardMetadata diff --git a/src/exo/engines/mlx/utils_mlx.py b/src/exo/engines/mlx/utils_mlx.py index 60a21e30..955cbb88 100644 --- a/src/exo/engines/mlx/utils_mlx.py +++ b/src/exo/engines/mlx/utils_mlx.py @@ -5,14 +5,14 @@ import resource from asyncio import AbstractEventLoop from typing import Any, Callable -import mlx.core as mx -import mlx.nn as nn from mlx_lm.generate import stream_generate # type: ignore from mlx_lm.sample_utils import make_sampler from mlx_lm.tokenizer_utils import TokenizerWrapper, load_tokenizer # type: ignore from mlx_lm.utils import load_model # type: ignore from pydantic import RootModel +import mlx.core as mx +import mlx.nn as nn # pyright: ignore[reportMissingTypeStubs] from exo.engines.mlx.auto_parallel import auto_parallel from exo.shared.types.api import ChatCompletionMessage from exo.shared.types.common import Host @@ -117,8 +117,10 @@ async def apply_chat_template( formatted_messages = [] for message in messages_dicts: filtered_message: dict[str, Any] = { - k: v for k, v in message.items() if v is not None - } # type: ignore + k: v + for k, v in message.items() # pyright: ignore[reportAny] + if v is not None + } # Verify we have required fields if "role" not in filtered_message: diff --git a/src/exo/master/forwarder_supervisor.py b/src/exo/master/forwarder_supervisor.py index 50798c8a..a1fb6120 100644 --- a/src/exo/master/forwarder_supervisor.py +++ b/src/exo/master/forwarder_supervisor.py @@ -113,6 +113,7 @@ class ForwarderSupervisor: str(self._binary_path), "--events-db", str(EXO_WORKER_EVENT_DB), + # pair arguments f"{pairs}", stdout=None, stderr=None, diff --git a/src/exo/master/main.py b/src/exo/master/main.py index c0709db2..e7f982cb 100644 --- a/src/exo/master/main.py +++ b/src/exo/master/main.py @@ -14,6 +14,7 @@ from exo.shared.apply import apply from exo.shared.db.sqlite.config import EventLogConfig from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage from exo.shared.db.sqlite.event_log_manager import EventLogManager +from exo.shared.keypair import Keypair, get_node_id_keypair from exo.shared.types.common import CommandId, NodeId from exo.shared.types.events import ( Event, @@ -34,7 +35,6 @@ from exo.shared.types.events.commands import ( from exo.shared.types.state import State from exo.shared.types.tasks import ChatCompletionTask, TaskId, TaskStatus, TaskType from exo.shared.types.worker.instances import Instance -from exo.shared.utils import Keypair, get_node_id_keypair class Master: @@ -263,8 +263,8 @@ async def async_main(): command_buffer, global_events, worker_events, - Path(os.environ["GO_BUILD_DIR"]) / "forwarder", - logger, + forwarder_binary_path=Path(os.environ["GO_BUILD_DIR"]) / "forwarder", + logger=logger, ) await master.run() diff --git a/src/exo/master/tests/test_forwarder_supervisor.py b/src/exo/master/tests/test_forwarder_supervisor.py index 00829696..1ac45bbd 100644 --- a/src/exo/master/tests/test_forwarder_supervisor.py +++ b/src/exo/master/tests/test_forwarder_supervisor.py @@ -393,6 +393,6 @@ class TestElectionCallbacks: callbacks = ElectionCallbacks(mock_supervisor, test_logger) await callbacks.on_became_replica() - mock_supervisor.notify_role_change.assert_called_once_with( + mock_supervisor.notify_role_change.assert_called_once_with( # type: ignore ForwarderRole.REPLICA - ) # type: ignore + ) diff --git a/src/exo/master/tests/test_master.py b/src/exo/master/tests/test_master.py index 293e454d..5e63ce52 100644 --- a/src/exo/master/tests/test_master.py +++ b/src/exo/master/tests/test_master.py @@ -10,6 +10,7 @@ from exo.master.main import Master from exo.shared.db.sqlite.config import EventLogConfig from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage from exo.shared.db.sqlite.event_log_manager import EventLogManager +from exo.shared.keypair import Keypair from exo.shared.types.api import ChatCompletionMessage, ChatCompletionTaskParams from exo.shared.types.common import NodeId from exo.shared.types.events import Event, EventFromEventLog, Heartbeat, TaskCreated @@ -31,10 +32,13 @@ from exo.shared.types.profiling import ( SystemPerformanceProfile, ) from exo.shared.types.tasks import ChatCompletionTask, TaskStatus, TaskType -from exo.shared.types.worker.common import InstanceId -from exo.shared.types.worker.instances import Instance, InstanceStatus, ShardAssignments +from exo.shared.types.worker.instances import ( + Instance, + InstanceId, + InstanceStatus, + ShardAssignments, +) from exo.shared.types.worker.shards import PartitionStrategy, PipelineShardMetadata -from exo.shared.utils import Keypair def _create_forwarder_dummy_binary() -> Path: diff --git a/src/exo/shared/constants.py b/src/exo/shared/constants.py index fe1393c3..eb7b7ba9 100644 --- a/src/exo/shared/constants.py +++ b/src/exo/shared/constants.py @@ -16,6 +16,8 @@ EXO_NODE_ID_KEYPAIR = EXO_HOME / "node_id.keypair" EXO_WORKER_KEYRING_FILE = EXO_HOME / "worker_keyring" EXO_MASTER_KEYRING_FILE = EXO_HOME / "master_keyring" +EXO_IPC_DIR = EXO_HOME / "ipc" + # libp2p topics for event forwarding LIBP2P_WORKER_EVENTS_TOPIC = "worker_events" LIBP2P_GLOBAL_EVENTS_TOPIC = "global_events" diff --git a/src/exo/shared/db/sqlite/event_log_manager.py b/src/exo/shared/db/sqlite/event_log_manager.py index 9a1aa1d9..571d6c8c 100644 --- a/src/exo/shared/db/sqlite/event_log_manager.py +++ b/src/exo/shared/db/sqlite/event_log_manager.py @@ -7,6 +7,7 @@ from sqlalchemy.exc import OperationalError from exo.shared.constants import EXO_HOME from exo.shared.db.sqlite.config import EventLogConfig, EventLogType from exo.shared.db.sqlite.connector import AsyncSQLiteEventStorage +from exo.shared.utils.fs import ensure_directory_exists class EventLogManager: @@ -25,7 +26,7 @@ class EventLogManager: self._connectors: Dict[EventLogType, AsyncSQLiteEventStorage] = {} # Ensure base directory exists - EXO_HOME.mkdir(parents=True, exist_ok=True) + ensure_directory_exists(EXO_HOME) # TODO: This seems like it's a pattern to avoid an async __init__ function. But as we know, there's a better pattern for this - using a create() function, like in runner_supervisor. async def initialize(self, max_retries: int = 3) -> None: diff --git a/src/exo/shared/ipc/__init__.py b/src/exo/shared/ipc/__init__.py new file mode 100644 index 00000000..c6f0a7bd --- /dev/null +++ b/src/exo/shared/ipc/__init__.py @@ -0,0 +1,14 @@ +""" +A set of IPC primitives intended for cross-language use. +Includes things like file-locks, named-pipe duplexes, and so on. + +TODO: implement System V IPC primitives?? + 1. semaphores w/ SEM_UNDO flag ??? + 2. Message Queues => as a replacement for pipe duplexes??? + see: https://www.softprayog.in/programming/system-v-semaphores + https://tldp.org/LDP/lpg/node21.html + https://tldp.org/LDP/tlk/ipc/ipc.html + https://docs.oracle.com/cd/E19683-01/816-5042/auto32/index.html + https://www.softprayog.in/programming/posix-semaphores + +""" diff --git a/src/exo/shared/ipc/file_mutex/__init__.py b/src/exo/shared/ipc/file_mutex/__init__.py new file mode 100644 index 00000000..f8465963 --- /dev/null +++ b/src/exo/shared/ipc/file_mutex/__init__.py @@ -0,0 +1,4 @@ +""" +A file-lock based IPC mutex primitives. + +""" diff --git a/src/exo/shared/ipc/file_mutex/flock_mutex.py b/src/exo/shared/ipc/file_mutex/flock_mutex.py new file mode 100644 index 00000000..fda65d60 --- /dev/null +++ b/src/exo/shared/ipc/file_mutex/flock_mutex.py @@ -0,0 +1,147 @@ +""" +File-based mutex primitive implemented using UNIX-based `flock` syscall. + +""" + +import contextlib +import errno +import fcntl +import os +import stat +import time +from enum import Enum +from typing import Optional + +from exo.shared.utils.fs import StrPath, ensure_parent_directory_exists + +# open in read-write mode, creates file if it doesn't exist already, +# closes this file descriptor in any children processes (prevents FD leaking), +# truncates this file on opening (lock-files shouldn't hold content FOR NOW!!!) +# SEE: https://man7.org/linux/man-pages/man2/openat.2.html +OPEN_FLAGS = os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_TRUNC + +# 0x644 mode flags -> user has read-write permissions, others have read permission only +# SEE: https://man7.org/linux/man-pages/man2/openat.2.html +MODE_FLAGS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + +# default poll-interval for spin-blocking lock +POLL_INTERVAL = 0.05 + + +class LockType(Enum): + READ = fcntl.LOCK_SH + WRITE = fcntl.LOCK_EX + + +class AcquireMode(Enum): + OS_BLOCKING = 0 + SPIN_BLOCKING = 1 + NON_BLOCKING = 2 + + +class FlockMutex: + def __init__(self, file_path: StrPath): + self._file_path = file_path + self._fd: Optional[int] = None + self.lock_held: Optional[LockType] = None + + def _open_fd(self): + assert self._fd is None + ensure_parent_directory_exists(self._file_path) + + # open file & TRY to change permissions to `MODE_FLAGS` flags + self._fd = os.open(self._file_path, OPEN_FLAGS, MODE_FLAGS) + with contextlib.suppress( + PermissionError + ): # This locked is not owned by this UID + os.chmod(self._fd, MODE_FLAGS) + + def _close_fd(self): + assert self._fd is not None + os.close(self._fd) + self._fd = None + + def _acquire(self, lock_type: LockType, blocking: bool) -> bool: + assert (self._fd is not None) and (self.lock_held is None) + + # create flags for acquiring lock + flags = lock_type.value + if not blocking: + flags |= fcntl.LOCK_NB + + # continually try to acquire lock (since it may fail due to interrupts) + while True: + try: + fcntl.flock(self._fd, flags) + break + except OSError as e: + if e.errno == errno.EINTR: # call interrupted by signal -> try again + continue + elif ( + e.errno == errno.EWOULDBLOCK + ): # file is locked & non-blocking is enabled -> return false to indicate + return False + + # unhandleable errors -> close FD & raise + self._close_fd() + if e.errno == errno.ENOSYS: # NotImplemented error + raise NotImplementedError( + "This system doesn't support flock" + ) from e + else: + raise + + # set lock-type held + self.lock_held = lock_type + return True + + def _release(self): + assert (self._fd is not None) and (self.lock_held is not None) + + # continually try to release lock (since it may fail due to interrupts) + while True: + try: + fcntl.flock(self._fd, fcntl.LOCK_UN) + break + except OSError as e: + if e.errno == errno.EINTR: # call interrupted by signal -> try again + continue + + # unhandleable errors -> close FD & raise + self._close_fd() + if e.errno == errno.ENOSYS: # NotImplemented error + raise NotImplementedError( + "This system doesn't support flock" + ) from e + else: + raise + + self.lock_held = None + + def acquire( + self, + lock_type: LockType = LockType.WRITE, + acquire_mode: AcquireMode = AcquireMode.SPIN_BLOCKING, + ) -> bool: + if self._fd is None: + self._open_fd() + + # OS-blocking & non-blocking is direct passthrough to private function + match acquire_mode: + case AcquireMode.OS_BLOCKING: + return self._acquire(lock_type, blocking=True) + case AcquireMode.NON_BLOCKING: + return self._acquire(lock_type, blocking=False) + case _: + pass + + # spin-blocking works by trying to acquire the lock in non-blocking mode, and retrying until success + while True: + locked = self._acquire(lock_type, blocking=False) + if locked: + return True + time.sleep(POLL_INTERVAL) + + def release(self): + self._release() + self._close_fd() diff --git a/src/exo/shared/ipc/pipe_duplex.py b/src/exo/shared/ipc/pipe_duplex.py new file mode 100644 index 00000000..3ba5a98e --- /dev/null +++ b/src/exo/shared/ipc/pipe_duplex.py @@ -0,0 +1,415 @@ +""" +SEE: + - https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html + - https://man7.org/linux/man-pages/man2/openat.2.html + - https://man7.org/linux/man-pages/man3/mkfifo.3.html + - https://man7.org/linux/man-pages/man7/pipe.7.html + +TODO: add locking on reader/writer ends to prevent multiwriters?? +TODO: use signal bytes to ensure proper packet consistency + +stretch: implement packet IDs, retries, dual-stream confirmations, RPCs & so on + +TODO: for more hardening -> check if any of the syscalls used return signal interrupt errors (like in the locking case) + and interrupt on that happening -> this may not be an issue PER SE but might potentially create insanely bizzare bugs + if it happens that this behavior DOES occasionally happen for no apparent reason + +TODO: maybe consider padding all messages with 0s on both ends ?? so as to prevent ANY ambiguous boundaries ever!! +""" + +import errno +import logging +import multiprocessing +import os +import queue +import stat +import threading +import time +from enum import Enum +from multiprocessing.queues import Queue as MQueueT +from multiprocessing.synchronize import Event as MEventT +from threading import Event as TEventT +from typing import Callable + +from cobs import cobs # pyright: ignore[reportMissingTypeStubs] +from pytest import LogCaptureFixture + +from exo.shared.utils.fs import ( + StrPath, + delete_if_exists, + ensure_parent_directory_exists, +) + +OPEN_READER_FLAGS = os.O_RDONLY | os.O_NONBLOCK +OPEN_WRITER_FLAGS = os.O_WRONLY | os.O_NONBLOCK + +# 0x644 mode flags -> user has read-write permissions, others have read permission only +MODE_FLAGS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + +POLL_INTERVAL = 0.05 # TODO: maybe parametrize this in classes?? +PIPE_BUF = 4096 # size of atomic writes on (most) UNIX pipes + + +class SignalMessage(Enum): + """ + Signal messages range from 1 to 255 & indicate control flow for the bytestream of the pipe. + + """ + + DISCARD_PREVIOUS = b"\x01" + + +class PipeDuplex: + """ + Creates a named-pipe communication duplex. The reader end is responsible for creating the pipe. + + The layers are: + 1. Raw binary data over pipes + 2. Variable-length binary packets with COBS + 3. JSON-like values with Message Pack + """ + + def __init__( + self, + in_pipe: StrPath, + out_pipe: StrPath, + in_callback: Callable[[bytes], None], + ): + assert in_pipe != out_pipe # they must be different files + + # pipes should only ever be created, and only by the reader (one-way operations) + _ensure_fifo_exists(in_pipe) # ensures reader pipe exists + + # create readonly properties (useful for inspection) + self._in_pipe = in_pipe + self._out_pipe = out_pipe + + # init synchronisation variables + self._mkill = multiprocessing.Event() + self._tkill = threading.Event() + in_mq: MQueueT[bytes] = multiprocessing.Queue() + self._out_mq: MQueueT[bytes] = multiprocessing.Queue() + in_mstarted = multiprocessing.Event() + + # process for reading in binary messages from pipe + self._p_in = multiprocessing.Process( + target=_pipe_buffer_reader, + args=(in_pipe, in_mq, in_mstarted, self._mkill), + daemon=True, + ) + self._p_in.start() + + # thread for pulling down binary messages from message queue & calling the callback + self._t_in = threading.Thread( + target=_binary_object_dispatcher, + args=(in_mq, in_callback, self._tkill), + daemon=True, + ) + self._t_in.start() + + # process to write binary messages to pipe + out_mstarted = multiprocessing.Event() + self._p_out = multiprocessing.Process( + target=_pipe_buffer_writer, + args=(out_pipe, self._out_mq, out_mstarted, self._mkill), + daemon=True, + ) + self._p_out.start() + + # wait for processes to start properly + in_mstarted.wait() + out_mstarted.wait() + + def __del__(self): + # signal to these processes to die (if they haven't already) + self._mkill.set() + self._tkill.set() + + def send_message(self, msg: bytes): + self._out_mq.put_nowait(msg) + + @property + def in_pipe(self): + return self._in_pipe + + @property + def out_pipe(self): + return self._out_pipe + + +def _ensure_fifo_exists(path: StrPath): + # try to make a file if one doesn't exist already + ensure_parent_directory_exists(path) + try: + os.mkfifo(path, mode=MODE_FLAGS) + except OSError as e: + # misc error, do not handle + if e.errno != errno.EEXIST: + raise + + # ensure the file exists is FIFO + st = os.stat(path) + if stat.S_ISFIFO(st.st_mode): + return + + # this file is not FIFO + raise FileExistsError(f"The file '{path}' isn't a FIFO") from e + + +def _pipe_buffer_reader( + path: StrPath, mq: MQueueT[bytes], started: MEventT, kill: MEventT +): + # TODO: right now the `kill` control flow is somewhat haphazard -> ensure every loop-y or blocking part always + # checks for kill.is_set() and returns/cleans up early if so + + # open reader in nonblocking mode -> should not fail & immediately open; + # this marks when the writer process has "started" + fd = os.open(path, OPEN_READER_FLAGS) + started.set() + print("(reader):", "started") + + # continually pull from the pipe and interpret messages as such: + # - all messages are separated/framed by NULL bytes (zero) + # - messages with >=2 bytes are COBS-encoded messages, because + # the smallest COBS-encoded message is 2 bytes + # - 1-byte messages are therefore to be treated as control signals + # + # TODO: right now i just need to get this to work, but the scheme is fundamentally + # extensible for robustness, e.g. signal-bytes can be used to drive state-machines + # for ensuring message atomicity/transmission + # e.g. we can use single-bytes to discriminate COBS values to say "this is length of upcoming message" + # vs. this is the actual content of the message, and so on + # . + # BUT for now we can just use signal (0xff 0x00) to mean "discard previous message" or similar... + # . + # BUT in THEORY we very well could have something like + # (0x10 0x00)[header signal] + (...)[header data like length & so on] + # + (0x20 0x00)[body signal] + (...)[body data] + # + (0x30 0x00)[checksum signal] + (...)[checksum data] + # And requests to re-send messages that were lost, and so on, like this is a fully 2-layer duplex + # communication so we could turn this into a VERY powerful thing some time in the future, like + # a whole-ass reimplementation of TCP/PIPES lmaooooo + buffer = bytearray() + while not kill.is_set(): + try: + # read available data (and try again if nothing) + try: + data = os.read(fd, PIPE_BUF) + if data == b"": + time.sleep(POLL_INTERVAL) + continue + except OSError as e: + if e.errno != errno.EAGAIN: + raise + + # if there is a writer connected & the buffer is empty, this would block + # so we must consume this error gracefully and try again + time.sleep(POLL_INTERVAL) + continue + + # extend buffer with new data + buffer.extend(data) + + # if there are no NULL bytes in the buffer, no new message has been formed + chunks = buffer.split(sep=b"\x00") + if len(chunks) == 1: + continue + + # last chunk is always an unfinished message, so that becomes our new buffer; + # the rest should be decoded as either signals or COBS and put on queue + buffer = chunks.pop() + for chunk in chunks: + chunk = bytes(chunk) + + # ignore empty messages (they mean nothing) + if chunk == b"": + continue + + # interpret 1-byte messages as signals (they indicate control-flow on messages) + if len(chunk) == 1: + print("(reader):", f"gotten control signal: {chunk[0]}") + continue # TODO: right now they should be ignored, since I'm not sure what I want them to do + + # interpret >=2 byte messages as COBS-encoded data (decode them) + decoded = cobs.decode(chunk) # pyright: ignore[reportUnknownMemberType] + mq.put(decoded) + except BaseException as e: + # perform cleanup & log before re-raising + os.close(fd) + logging.error(msg=f"Error when reading from named pipe at '{path}': {e}") + raise + os.close(fd) + + +def _binary_object_dispatcher( + mq: MQueueT[bytes], callback: Callable[[bytes], None], kill: TEventT +): + while not kill.is_set(): + # try to get with timeout (to allow to read the kill-flag) + try: + message = mq.get(block=True, timeout=POLL_INTERVAL) + except queue.Empty: + continue + + # dispatch binary object with callback + callback(message) + + +def _pipe_buffer_writer( + path: StrPath, mq: MQueueT[bytes], started: MEventT, kill: MEventT +): + # TODO: right now the `kill` control flow is somewhat haphazard -> ensure every loop-y or blocking part always + # checks for kill.is_set() and returns/cleans up early if so + + # for now, started events for writer are rather vacuous: TODO: remove or make more usefull?? + started.set() + print("(writer):", "started") + + # continually attempt to open FIFO for reading in nonblocking mode -> will error that: + # - ENOENT[2] No such file or directory: until a reader creates FIFO + # - ENXIO[6] No such device or address: until a reader opens FIFO + fd = None + while not kill.is_set(): + try: + fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK) + + # ensure the file exists is FIFO + st = os.fstat(fd) + print("mode:", st.st_mode & 0o170000) + if stat.S_ISFIFO(st.st_mode): + break + + # cleanup on error + os.close(fd) + raise FileExistsError(f"The file '{path}' isn't a FIFO") + except FileExistsError: + raise # propagate error + except OSError as e: + # misc error, do not handle + if not (e.errno == errno.ENOENT or e.errno == errno.ENXIO): + raise + + # try again if waiting for FIFO creation or reader-end opening + time.sleep(POLL_INTERVAL) + continue + assert fd is not None + + while not kill.is_set(): + try: + # try to get with timeout (to allow to read the kill-flag) + try: + data = mq.get(block=True, timeout=POLL_INTERVAL) + except queue.Empty: + continue + + # write all data (by continually re-trying until it is done) + _write_data(fd, data) + except BaseException as e: + # perform cleanup & log before re-raising + os.close(fd) + logging.error(msg=f"Error when writing to named pipe at '{path}': {e}") + raise + + os.close(fd) + + +def _write_data(fd: int, buf: bytes): + # COBS-encode the data & append NULL-byte to signify end-of-frame + buf = cobs.encode(buf) + b"\x00" # pyright: ignore[reportUnknownMemberType] + total = len(buf) + sent = 0 + + # begin transmission progress + while sent < total: + try: + # Write remaining bytes to the pipe + written = os.write(fd, buf[sent:]) + sent += written + except OSError as e: + # non-blocking pipe is full, wait a bit and retry + if e.errno == errno.EAGAIN: + time.sleep(POLL_INTERVAL) + continue + + # reader disconnected -> handle failure-recovery by doing: + # 1. signal DISCARD_PREVIOUS to any reader + # 2. re-setting the progress & trying again + if e.errno == errno.EPIPE: + _write_signal(fd, SignalMessage.DISCARD_PREVIOUS) + sent = 0 + continue + + raise # misc error, do not handle + + +def _write_signal(fd: int, signal: SignalMessage): + signal_message_length = 2 + + # Turn signal-byte into message by terminating with NULL-byte + buf = signal.value + b"\x00" + assert len(buf) == signal_message_length + + # attempt to write until successful + while True: + try: + # small writes (e.g. 2 bytes) should be atomic as per Pipe semantics, + # meaning IF SUCCESSFUL: the number of bytes written MUST be exactly 2 + written = os.write(fd, buf) + assert written == signal_message_length + break + except OSError as e: + # wait a bit and retry if: + # - non-blocking pipe is full + # - the pipe is broken because of reader disconnection + if e.errno == errno.EAGAIN or e.errno == errno.EPIPE: + time.sleep(POLL_INTERVAL) + continue + + raise # misc error, do not handle + + +def _test_one_two_three(): + one_path = "/tmp/one.pipe" + two_path = "/tmp/two.pipe" + delete_if_exists(one_path) + delete_if_exists(two_path) + + owner = PipeDuplex( + in_pipe=one_path, + out_pipe=two_path, + in_callback=lambda x: print(f"wow, owner got: [{len(x)}]{x}"), + ) + + guest = PipeDuplex( + in_pipe=two_path, + out_pipe=one_path, + in_callback=lambda x: print(f"wow, guest1 got: [{len(x)}]{x}"), + ) + + owner.send_message(bytes(0 for _ in range(10))) + + guest.send_message(bytes(0 for _ in range(200))) + + time.sleep(1) + + del guest + guest = PipeDuplex( + in_pipe=two_path, + out_pipe=one_path, + in_callback=lambda x: print(f"wow, guest2 got: [{len(x)}]{x}"), + ) + + guest.send_message(bytes(0 for _ in range(21))) + + owner.send_message(bytes(0 for _ in range(12))) + + time.sleep(1) + + delete_if_exists(one_path) + delete_if_exists(two_path) + + +def test_running_pipe_duplex(caplog: LogCaptureFixture): + caplog.set_level(logging.INFO) + + _test_one_two_three() + time.sleep(1) diff --git a/src/exo/shared/utils.py b/src/exo/shared/keypair.py similarity index 97% rename from src/exo/shared/utils.py rename to src/exo/shared/keypair.py index a819e7fb..a78c2cb4 100644 --- a/src/exo/shared/utils.py +++ b/src/exo/shared/keypair.py @@ -4,7 +4,7 @@ import hashlib import logging import os from pathlib import Path -from typing import Any, Type, final +from typing import final import base58 from cryptography.hazmat.primitives import serialization @@ -216,12 +216,6 @@ class Keypair: return self._public_key -def ensure_type[T](obj: Any, expected_type: Type[T]) -> T: # type: ignore - if not isinstance(obj, expected_type): - raise TypeError(f"Expected {expected_type}, got {type(obj)}") # type: ignore - return obj - - def get_node_id_keypair( path: str | bytes | os.PathLike[str] | os.PathLike[bytes] = EXO_NODE_ID_KEYPAIR, ) -> Keypair: diff --git a/src/exo/shared/tests/test_flock_mutex.py b/src/exo/shared/tests/test_flock_mutex.py new file mode 100644 index 00000000..42d68753 --- /dev/null +++ b/src/exo/shared/tests/test_flock_mutex.py @@ -0,0 +1,48 @@ +import pytest + +from exo.shared.ipc.file_mutex.flock_mutex import FlockMutex, LockType +from exo.shared.utils.fs import delete_if_exists, make_temp_path + + +def test_lock_held(): + path = make_temp_path("testing_flock.lock") + lock = FlockMutex(path) + + assert lock.lock_held is None + + assert lock.acquire(lock_type=LockType.WRITE) + assert lock.lock_held == LockType.WRITE + lock.release() + + assert lock.lock_held is None + + assert lock.acquire(lock_type=LockType.READ) + assert lock.lock_held == LockType.READ + lock.release() + + assert lock.lock_held is None + + delete_if_exists(path) + + +def test_no_reentrant_lock(): + path = make_temp_path("testing_flock.lock") + lock = FlockMutex(path) + + # no write-lock reentrancy + lock.acquire(lock_type=LockType.WRITE) + with pytest.raises(AssertionError): + lock.acquire(lock_type=LockType.WRITE) + with pytest.raises(AssertionError): + lock.acquire(lock_type=LockType.READ) + lock.release() + + # no read-lock reentrancy + lock.acquire(lock_type=LockType.READ) + with pytest.raises(AssertionError): + lock.acquire(lock_type=LockType.WRITE) + with pytest.raises(AssertionError): + lock.acquire(lock_type=LockType.READ) + lock.release() + + delete_if_exists(path) diff --git a/src/exo/shared/tests/test_node_id_persistence.py b/src/exo/shared/tests/test_node_id_persistence.py index 552311e7..46a81d55 100644 --- a/src/exo/shared/tests/test_node_id_persistence.py +++ b/src/exo/shared/tests/test_node_id_persistence.py @@ -14,7 +14,7 @@ from typing import Optional from pytest import LogCaptureFixture from exo.shared.constants import EXO_NODE_ID_KEYPAIR -from exo.shared.utils import get_node_id_keypair +from exo.shared.keypair import get_node_id_keypair NUM_CONCURRENT_PROCS = 10 diff --git a/src/exo/shared/types/events/_events.py b/src/exo/shared/types/events/_events.py index c59a2df1..dccb9f6f 100644 --- a/src/exo/shared/types/events/_events.py +++ b/src/exo/shared/types/events/_events.py @@ -297,8 +297,8 @@ def _check_event_type_consistency(): # grab type hints and extract the right values from it cls_hints = get_type_hints(cls) assert ( - "event_type" in cls_hints and get_origin(cls_hints["event_type"]) is Literal - ), ( # pyright: ignore[reportAny] + "event_type" in cls_hints and get_origin(cls_hints["event_type"]) is Literal # type: ignore + ), ( f"{get_error_reporting_message()}", f"The class {cls} is missing a {Literal}-annotated `event_type` field.", ) diff --git a/src/exo/shared/utils/__init__.py b/src/exo/shared/utils/__init__.py new file mode 100644 index 00000000..87131484 --- /dev/null +++ b/src/exo/shared/utils/__init__.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import Any, Type + +from exo.shared.utils.phantom import PhantomData + + +def ensure_type[T](obj: Any, expected_type: Type[T]) -> T: # type: ignore + if not isinstance(obj, expected_type): + raise TypeError(f"Expected {expected_type}, got {type(obj)}") # type: ignore + return obj + + +def todo[T]( + msg: str = "This code has not been implemented yet.", + _phantom: PhantomData[T] = None, +) -> T: + raise NotImplementedError(msg) diff --git a/src/exo/shared/utils/fs.py b/src/exo/shared/utils/fs.py new file mode 100644 index 00000000..a72a73ba --- /dev/null +++ b/src/exo/shared/utils/fs.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import contextlib +import os +import pathlib +import tempfile +from typing import LiteralString + +type StrPath = str | os.PathLike[str] +type BytesPath = bytes | os.PathLike[bytes] +type StrOrBytesPath = str | bytes | os.PathLike[str] | os.PathLike[bytes] + + +def delete_if_exists(filename: StrOrBytesPath) -> None: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + +def ensure_parent_directory_exists(filename: StrPath) -> None: + """ + Ensure the directory containing the file exists (create it if necessary). + """ + pathlib.Path(filename).parent.mkdir(parents=True, exist_ok=True) + + +def ensure_directory_exists(dirname: StrPath) -> None: + """ + Ensure the directory exists (create it if necessary). + """ + pathlib.Path(dirname).mkdir(parents=True, exist_ok=True) + + +def make_temp_path(name: LiteralString) -> str: + return os.path.join(tempfile.mkdtemp(), name) diff --git a/src/exo/shared/utils/phantom.py b/src/exo/shared/utils/phantom.py new file mode 100644 index 00000000..7311ea6e --- /dev/null +++ b/src/exo/shared/utils/phantom.py @@ -0,0 +1,14 @@ +from typing import Optional + + +class _PhantomData[T]: + """ + Internal machinery of the phantom data - it stores nothing. + """ + + +type PhantomData[T] = Optional[_PhantomData[T]] +""" +Allows you to use generics in functions without storing anything of that generic type. +Just use `None` and you'll be fine +""" diff --git a/src/exo/shared/utils/pydantic_ext.py b/src/exo/shared/utils/pydantic_ext.py new file mode 100644 index 00000000..e85591f7 --- /dev/null +++ b/src/exo/shared/utils/pydantic_ext.py @@ -0,0 +1,52 @@ +from pydantic import BaseModel +from pydantic.alias_generators import to_camel + + +class CamelCaseModel(BaseModel): + """ + A model whose fields are aliased to camel-case from snake-case. + """ + + class Config: + alias_generator = to_camel + allow_population_by_field_name = True + + +class Tagged[Tag: str, Content]( + CamelCaseModel +): # TODO: figure out how to make pydantic work with LiteralString + """ + Utility for helping with serializing unions as adjacently tagged with Pydantic. + + By default, Pydantic uses internally tagged union ser/de BUT to play nicely with + other cross-language ser/de tools, you need adjacently tagged unions, and Pydantic + doesn't support those out of the box. + + SEE: https://serde.rs/enum-representations.html#adjacently-tagged + + Example usage: + ```python + TaggedUnion = Annotated[Union[ + Tagged[Literal["Foo"], Foo], + Tagged[Literal["Bar"], Bar] + ], Field(discriminator="t")] + + Parser: TypeAdapter[TaggedUnion] = TypeAdapter(TaggedUnion) + + def validate_python(v: any) -> Foo | Bar: + v = Parser.validate_python(v) + match v.t: + case "Foo": return v.c + case "Bar": return v.c + ``` + """ + + t: Tag + """ + The tag corresponding to the type of the object in the union. + """ + + c: Content + """ + The actual content of the object of that type. + """ diff --git a/src/exo/shared/utils/reactive.py b/src/exo/shared/utils/reactive.py new file mode 100644 index 00000000..14c021d2 --- /dev/null +++ b/src/exo/shared/utils/reactive.py @@ -0,0 +1,32 @@ +""" +Utilities for reactive variables + +""" + +from typing import Protocol + + +class OnChange[T](Protocol): + def __call__(self, old_value: T, new_value: T) -> None: ... + + +class Reactive[T]: + def __init__(self, initial_value: T, on_change: OnChange[T]): + self._value = initial_value + self._on_change = on_change + + @property + def value(self): + return self._value + + @value.setter + def value(self, new_value: T): + old_value = self._value + self._value = new_value + + # don't notify when not changed + if old_value == new_value: + return + + # notify of changes + self._on_change(old_value=old_value, new_value=new_value) diff --git a/src/exo/worker/main.py b/src/exo/worker/main.py index 621d0cc1..abd9af78 100644 --- a/src/exo/worker/main.py +++ b/src/exo/worker/main.py @@ -3,6 +3,7 @@ import logging from exo.shared.apply import apply from exo.shared.db.sqlite.event_log_manager import EventLogConfig, EventLogManager +from exo.shared.keypair import Keypair, get_node_id_keypair from exo.shared.types.common import NodeId from exo.shared.types.events import ( NodePerformanceMeasured, @@ -12,7 +13,6 @@ from exo.shared.types.worker.ops import ( ExecuteTaskOp, RunnerOp, ) -from exo.shared.utils import Keypair, get_node_id_keypair from exo.worker.download.impl_shard_downloader import exo_shard_downloader from exo.worker.plan import plan from exo.worker.utils.profile import start_polling_node_metrics diff --git a/src/exo/worker/runner/runner.py b/src/exo/worker/runner/runner.py index 25e1a025..440dcdef 100644 --- a/src/exo/worker/runner/runner.py +++ b/src/exo/worker/runner/runner.py @@ -6,7 +6,7 @@ from functools import partial from typing import Callable, cast import mlx.core as mx -import mlx.nn as nn +import mlx.nn as nn # pyright: ignore [reportMissingTypeStubs] from mlx_lm.generate import stream_generate # type: ignore from mlx_lm.tokenizer_utils import TokenizerWrapper diff --git a/src/exo/worker/tests/test_spinup_timeout.py b/src/exo/worker/tests/test_spinup_timeout.py index 8649fef9..501ca649 100644 --- a/src/exo/worker/tests/test_spinup_timeout.py +++ b/src/exo/worker/tests/test_spinup_timeout.py @@ -37,9 +37,9 @@ async def test_runner_up_op_timeout( # _execute_runner_up_op should throw a TimeoutError with a short timeout events: list[Event] = [] - async for event in worker._execute_runner_up_op( + async for event in worker._execute_runner_up_op( # type: ignore[misc] runner_up_op, initialize_timeout=0.2 - ): # type: ignore[misc] + ): events.append(event) assert isinstance(events[-1], RunnerStatusUpdated) diff --git a/uv.lock b/uv.lock index f8fb5d9f..bb4af869 100644 --- a/uv.lock +++ b/uv.lock @@ -130,15 +130,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/45/ec96b29162a402fc4c1c5512d114d7b3787b9d1c2ec241d9568b4816ee23/base58-2.1.1-py3-none-any.whl", hash = "sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2", size = 5621, upload-time = "2021-10-30T22:12:16.658Z" }, ] -[[package]] -name = "braq" -version = "0.0.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/3b/1b918c408e11ca33f9b9dcecc8e08eac7762887dd42b584f0efb6fe26c55/braq-0.0.12.tar.gz", hash = "sha256:51dae51b863cbba2cd37da163df06b7dc5124904d2c26b92bda54c1bde66d74b", size = 15272, upload-time = "2024-12-10T20:48:53.856Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/53/ed5082619966b1d15b5c039ac722ba99956d92d4b08a9bd5eb4c3535cc1f/braq-0.0.12-py3-none-any.whl", hash = "sha256:41b7bdd0d004faef693751615fbb11c53ac0b886c772b83aea61ea6dc2f6e518", size = 26392, upload-time = "2024-12-10T20:48:50.813Z" }, -] - [[package]] name = "certifi" version = "2025.8.3" @@ -204,6 +195,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, ] +[[package]] +name = "cobs" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/ef/ea149311227a4fc3160cc885fce06da7c7d76782a308ef070b8065c69953/cobs-1.2.2.tar.gz", hash = "sha256:dbdd5e32111d72786f83d0c269215dcd6ac629b1ac1962c6878221f3b2ca98da", size = 14582, upload-time = "2025-07-20T01:08:35.434Z" } + [[package]] name = "cryptography" version = "45.0.6" @@ -253,6 +250,7 @@ dependencies = [ { name = "aiohttp", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "aiosqlite", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "base58", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "cobs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -294,6 +292,7 @@ requires-dist = [ { name = "aiohttp", specifier = ">=3.12.14" }, { name = "aiosqlite", specifier = ">=0.21.0" }, { name = "base58", specifier = ">=2.1.1" }, + { name = "cobs", specifier = ">=1.2.2" }, { name = "cryptography", specifier = ">=45.0.5" }, { name = "fastapi", specifier = ">=0.116.1" }, { name = "filelock", specifier = ">=3.18.0" }, @@ -331,14 +330,14 @@ name = "exo-scripts" version = "0.1.0" source = { editable = "scripts" } dependencies = [ + { name = "exo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "shared", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.metadata] requires-dist = [ + { name = "exo", editable = "." }, { name = "huggingface-hub", specifier = ">=0.33.4" }, - { name = "shared" }, ] [[package]] @@ -566,19 +565,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" }, ] -[[package]] -name = "kvf" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "braq", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "paradict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9c/f8/e1826c156d4f97cf4662a6110cbbcfd91b5e5570c8a88bf0a8270718621e/kvf-0.0.3.tar.gz", hash = "sha256:f4885b1bbe66c8c20fdabe5cedeb3c0e5d12a54ac495f9e5fcf6fed0e0c51b73", size = 4938, upload-time = "2024-12-10T20:49:13.171Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/db/4a8d3b1fef45cabcadf36f9a2231b2cde3dddd3a58ab1723119c7fbce34f/kvf-0.0.3-py3-none-any.whl", hash = "sha256:9d666e51cae512e3f95c55b77524e34d0095b278c81f96f7bbc7d37b5bd545c6", size = 4716, upload-time = "2024-12-10T20:49:11.815Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -764,15 +750,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "paradict" -version = "0.0.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/83/8cf8d94be55ab9ea783e1f8ece06059cd986bb482ad69f7be549839b9e07/paradict-0.0.16.tar.gz", hash = "sha256:d909d122bf47028a45334eb2280d1e1bcb401fda89986af42c39fd2fadf9de4d", size = 61471, upload-time = "2024-12-10T21:23:49.007Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/f9/a9807d307ba1837bb8799e1337f41edcdbb92ef6090668dc50f483a168bf/paradict-0.0.16-py3-none-any.whl", hash = "sha256:28df79f0dc0e68c8f8a3e9b7c75e67a85305ef7298653fc7a369a1bf4f58cb20", size = 61735, upload-time = "2024-12-10T21:23:45.408Z" }, -] - [[package]] name = "pathlib" version = "1.0.1" @@ -1072,19 +1049,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957, upload-time = "2025-08-08T13:13:57.029Z" }, ] -[[package]] -name = "shared" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "kvf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "paradict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3f/39/f39c2560ac971efbf437f7ffa1d82a12fa77f50b0127e6e5ec5cc8d377df/shared-0.0.32.tar.gz", hash = "sha256:7308adc95c0dab14d0c99635cd8049d1f004cc7fef7396d3fe47323c34ec58c6", size = 7793, upload-time = "2024-12-10T20:49:22.469Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/03/da58e40386d8ebcdfa3617070a95ca1deb5a5e6aa3d4e15ea2045173d5ac/shared-0.0.32-py3-none-any.whl", hash = "sha256:f17962c0f0fe6a23015accc7cac029e1c24c4b14578094e1f7033a7a7ef16140", size = 29304, upload-time = "2024-12-10T20:49:19.763Z" }, -] - [[package]] name = "sniffio" version = "1.3.1" diff --git a/worker/pyproject.toml b/worker/pyproject.toml deleted file mode 100644 index dca88c33..00000000 --- a/worker/pyproject.toml +++ /dev/null @@ -1,18 +0,0 @@ -[project] -name = "worker" -version = "0.1.0" -description = "Worker for the Exo project" -readme = "README.md" -requires-python = ">=3.13" -dependencies = [ - "shared", - "huggingface_hub>=0.33.4", - "mlx>=0.26.3", - "mlx-lm @ https://github.com/ml-explore/mlx-lm.git", - "psutil>=7.0.0", - "transformers>=4.55.0", -] - -[build-system] -requires = ["uv_build>=0.8.9,<0.9.0"] -build-backend = "uv_build"