Compare commits

..
Author SHA1 Message Date
Alex CheemaandClaude Opus 4.6 4a3b45a439 fix: point MLX dependency to exo-explore/mlx pipe-sidechannel fork
Replace personal fork (rltakashige/mlx-jaccl-fix-small-recv) with the
org-owned fork (exo-explore/mlx) on the pipe-sidechannel branch. This
branch contains the C++ SideChannel pipe mode that provides the
MLX_JACCL_PIPE_IN/MLX_JACCL_PIPE_OUT env var support needed by the
JACCL pipe relay.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-21 12:51:35 -08:00
Alex CheemaandClaude Opus 4.6 f984c190d5 feat: add JACCL SideChannel pipe relay for distributed tensor ops
Implement named-pipe (FIFO) based relay for JACCL all_gather operations
across the exo control plane, enabling distributed tensor operations
between MlxJaccl runner instances.

Components:
- Base64Bytes type + JacclSideChannelData/Gathered event types
- RunnerSupervisor: FIFO creation, _pipe_relay() async loop that reads
  local data from runner, emits events, waits for gathered result, and
  writes ordered data back
- Bootstrap: opens FIFOs in child process, sets MLX_JACCL_PIPE_IN/OUT
  env vars for C++ SideChannel
- Worker: routes JacclSideChannelGathered events to RunnerSupervisors
- Master: _handle_jaccl_side_channel() accumulates per-runner data and
  emits gathered event when all runners for an instance have contributed
- mx_any() docstring explaining all_sum for GPU deadlock prevention

Extracted from meta-instance branch (#1519) — PR 4 of 5.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-21 12:50:59 -08:00
725 changed files with 8915 additions and 50009 deletions

No files matched your search

+4 -124
View File
@@ -32,6 +32,7 @@ jobs:
SPARKLE_ED25519_PRIVATE: ${{ secrets.SPARKLE_ED25519_PRIVATE }}
SPARKLE_S3_BUCKET: ${{ secrets.SPARKLE_S3_BUCKET }}
SPARKLE_S3_PREFIX: ${{ secrets.SPARKLE_S3_PREFIX }}
EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT: ${{ secrets.EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT }}
AWS_REGION: ${{ secrets.AWS_REGION }}
EXO_BUILD_NUMBER: ${{ github.run_number }}
EXO_LIBP2P_NAMESPACE: ${{ github.ref_name }}
@@ -158,7 +159,7 @@ jobs:
fi
- name: Install Homebrew packages
run: brew install just awscli
run: brew install just awscli macmon
- name: Install UV
uses: astral-sh/setup-uv@v6
@@ -238,92 +239,10 @@ jobs:
# Export keychain path for other steps
echo "BUILD_KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV
# ============================================================
# Pre-flight credential / profile validation
# Runs BEFORE the ~16 min build so auth/expiry failures surface in <1 min.
# ============================================================
- name: Validate Apple notarization credentials
env:
APPLE_NOTARIZATION_USERNAME: ${{ secrets.APPLE_NOTARIZATION_USERNAME }}
APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
run: |
# All-or-nothing: either all three creds are set, or none are.
CRED_COUNT=0
for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
[[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
done
if [[ "$CRED_COUNT" -eq 0 ]]; then
echo "No notarization credentials configured — skipping notarization for this build."
exit 0
fi
if [[ "$CRED_COUNT" -ne 3 ]]; then
echo "ERROR: partial notarization credentials set ($CRED_COUNT/3). Aborting before build."
exit 1
fi
# Cheap, ~5s, auth-only call. Fails instantly with a clear message if
# the app-specific password is stale, wrong team-id, etc.
echo "Verifying Apple notarization credentials via notarytool history..."
if ! xcrun notarytool history \
--apple-id "$APPLE_NOTARIZATION_USERNAME" \
--password "$APPLE_NOTARIZATION_PASSWORD" \
--team-id "$APPLE_NOTARIZATION_TEAM" >/dev/null; then
echo "ERROR: notarytool rejected the provided credentials. Fix before rerunning."
echo "Common causes: app-specific password expired/revoked, wrong team-id,"
echo "Apple ID not on the team, or 2FA not configured for this Apple ID."
exit 1
fi
echo "Apple notarization credentials OK."
- name: Validate provisioning profile expiry
run: |
PROFILE="$HOME/Library/Developer/Xcode/UserData/Provisioning Profiles/EXO.provisionprofile"
if [[ ! -f "$PROFILE" ]]; then
echo "ERROR: provisioning profile not found at $PROFILE"
exit 1
fi
EXPIRY=$(security cms -D -i "$PROFILE" | plutil -extract ExpirationDate raw -o - - 2>/dev/null || true)
if [[ -z "$EXPIRY" ]]; then
echo "WARNING: could not read ExpirationDate from provisioning profile; skipping expiry check."
exit 0
fi
# Try a couple of known plutil date formats. If none parse, skip the check rather
# than risk a false-positive "expired" block on a format we didn't anticipate.
EXPIRY_EPOCH=""
for fmt in "%Y-%m-%dT%H:%M:%SZ" "%Y-%m-%d %H:%M:%S %z" "%Y-%m-%d %H:%M:%S +0000"; do
if parsed=$(date -j -f "$fmt" "$EXPIRY" +%s 2>/dev/null); then
EXPIRY_EPOCH="$parsed"
break
fi
done
if [[ -z "$EXPIRY_EPOCH" ]]; then
echo "WARNING: could not parse ExpirationDate '$EXPIRY'; skipping expiry check."
exit 0
fi
NOW_EPOCH=$(date +%s)
if [[ "$EXPIRY_EPOCH" -le "$NOW_EPOCH" ]]; then
echo "ERROR: provisioning profile expired on $EXPIRY. Regenerate it before rerunning."
exit 1
fi
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
echo "Provisioning profile valid until $EXPIRY ($DAYS_LEFT days remaining)."
if [[ "$DAYS_LEFT" -lt 14 ]]; then
echo "WARNING: profile expires in under 14 days — regenerate soon."
fi
# ============================================================
# Build the bundle
# ============================================================
- name: Add pinned macmon to PATH
run: |
MACMON_DIR=$(nix develop --command sh -c 'dirname $(which macmon)')
echo "Using macmon from: $MACMON_DIR"
echo "$MACMON_DIR" >> $GITHUB_PATH
# Remove any Homebrew macmon so PyInstaller can't accidentally pick it up
brew uninstall macmon 2>/dev/null || true
- name: Build PyInstaller bundle
run: uv run pyinstaller packaging/pyinstaller/exo.spec
@@ -346,6 +265,7 @@ jobs:
EXO_BUILD_COMMIT="$GITHUB_SHA" \
SPARKLE_FEED_URL="$SPARKLE_FEED_URL" \
SPARKLE_ED25519_PUBLIC="$SPARKLE_ED25519_PUBLIC" \
EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT="$EXO_BUG_REPORT_PRESIGNED_URL_ENDPOINT" \
CODE_SIGNING_IDENTITY="$SIGNING_IDENTITY" \
CODE_SIGN_INJECT_BASE_ENTITLEMENTS=YES
mkdir -p ../../output
@@ -378,41 +298,11 @@ jobs:
APPLE_NOTARIZATION_PASSWORD: ${{ secrets.APPLE_NOTARIZATION_PASSWORD }}
APPLE_NOTARIZATION_TEAM: ${{ secrets.APPLE_NOTARIZATION_TEAM }}
run: |
set -o pipefail
cd output
security unlock-keychain -p "$MACOS_CERTIFICATE_PASSWORD" "$BUILD_KEYCHAIN_PATH"
SIGNING_IDENTITY=$(security find-identity -v -p codesigning "$BUILD_KEYCHAIN_PATH" | awk -F '"' '{print $2}')
# Fail fast if notarization creds are partial. All-or-nothing.
CRED_COUNT=0
for v in "$APPLE_NOTARIZATION_USERNAME" "$APPLE_NOTARIZATION_PASSWORD" "$APPLE_NOTARIZATION_TEAM"; do
[[ -n "$v" ]] && CRED_COUNT=$((CRED_COUNT + 1))
done
if [[ "$CRED_COUNT" -ne 0 && "$CRED_COUNT" -ne 3 ]]; then
echo "ERROR: partial Apple notarization credentials set ($CRED_COUNT/3). Aborting."
exit 1
fi
/usr/bin/codesign --deep --force --timestamp --options runtime \
--sign "$SIGNING_IDENTITY" EXO.app
# Pre-flight: verify the signed app BEFORE building DMG and submitting to Apple.
# If this fails, notarization will fail too — cheap way to fail in seconds, not 15 minutes.
echo "===== codesign --verify EXO.app ====="
if ! /usr/bin/codesign --verify --deep --strict --verbose=2 EXO.app; then
echo "ERROR: EXO.app failed codesign verification. Dumping signing status of every executable:"
find EXO.app -type f \( -perm -111 -o -name "*.dylib" -o -name "*.so" -o -name "*.framework" \) -print0 |
while IFS= read -r -d '' f; do
printf -- '--- %s\n' "$f"
/usr/bin/codesign -dv --verbose=2 "$f" 2>&1 | sed 's/^/ /' || true
done
exit 1
fi
# Gatekeeper assessment. A failure here strongly predicts notarization rejection.
echo "===== spctl assessment (predicts notarization outcome) ====="
/usr/bin/spctl -a -vvv -t install EXO.app || echo "WARNING: spctl assessment failed — notarization is likely to fail too."
mkdir -p dmg-root
cp -R EXO.app dmg-root/
ln -s /Applications dmg-root/Applications
@@ -420,22 +310,12 @@ jobs:
hdiutil create -volname "EXO" -srcfolder dmg-root -ov -format UDZO "$DMG_NAME"
/usr/bin/codesign --force --timestamp --options runtime \
--sign "$SIGNING_IDENTITY" "$DMG_NAME"
echo "===== codesign --verify DMG ====="
if ! /usr/bin/codesign --verify --verbose=2 "$DMG_NAME"; then
echo "ERROR: DMG failed codesign verification."
exit 1
fi
if [[ -n "$APPLE_NOTARIZATION_USERNAME" ]]; then
echo "===== notarytool submit ====="
# `|| true` so set -e doesn't abort before we can echo output / fetch the log.
# We rely on the parsed STATUS below to decide pass/fail.
SUBMISSION_OUTPUT=$(xcrun notarytool submit "$DMG_NAME" \
--apple-id "$APPLE_NOTARIZATION_USERNAME" \
--password "$APPLE_NOTARIZATION_PASSWORD" \
--team-id "$APPLE_NOTARIZATION_TEAM" \
--wait --timeout 15m 2>&1) || true
--wait --timeout 15m 2>&1)
echo "$SUBMISSION_OUTPUT"
SUBMISSION_ID=$(echo "$SUBMISSION_OUTPUT" | awk 'tolower($1)=="id:" && $2 ~ /^[0-9a-fA-F-]+$/ {print $2; exit}')
+3
View File
@@ -91,6 +91,9 @@ jobs:
nix build .#metal-toolchain
fi
# Build mlx (depends on metal-toolchain)
nix build .#mlx
- name: Build all Nix outputs
run: |
nix flake show --json | jq -r '
+1 -6
View File
@@ -18,6 +18,7 @@ digest.txt
app/EXO/build/
dist/
# rust
target/
**/*.rs.bk
@@ -37,9 +38,3 @@ bench/**/*.json
# tmp
tmp/models
/build/exo
/.agents
/.claude/skills
/.claude
/.codex
skills-lock.json
+31
View File
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="EMPTY_MODULE" version="4">
<component name="FacetManager">
<facet type="Python" name="Python facet">
<configuration sdkName="Python 3.13 virtualenv at ~/Desktop/exo/.venv" />
</facet>
</component>
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/scripts/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/rust/exo_pyo3_bindings/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/rust/exo_pyo3_bindings/tests" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/rust/util/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/rust/networking/examples" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/rust/networking/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/rust/networking/tests" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/rust/system_custodian/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/.venv" />
<excludeFolder url="file://$MODULE_DIR$/.direnv" />
<excludeFolder url="file://$MODULE_DIR$/build" />
<excludeFolder url="file://$MODULE_DIR$/dist" />
<excludeFolder url="file://$MODULE_DIR$/.go_cache" />
<excludeFolder url="file://$MODULE_DIR$/rust/target" />
</content>
<orderEntry type="jdk" jdkName="Python 3.13 (exo)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Python 3.13 virtualenv at ~/Desktop/exo/.venv interpreter library" level="application" />
</component>
</module>
+1 -1
View File
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalDependencies">
<plugin id="al.aoli.intellijdirenv" />
<plugin id="systems.fehn.intellijdirenv" />
</component>
</project>
+14
View File
@@ -0,0 +1,14 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ourVersions">
<value>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="3.14" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>
+2 -2
View File
@@ -4,7 +4,7 @@
<option name="sdkName" value="Python 3.13 (exo)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13 (exo)" project-jdk-type="Python SDK" />
<component name="RuffConfiguration">
<option name="enabled" value="true" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>
</project>
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Loaded 100 of 725 files, more files were not shown because too many files have changed in this diff. Show more