From 49d4dbb0dd4e8bd355c3c3115d6bc5d203b0583f Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Wed, 5 Oct 2022 09:45:54 +0800 Subject: [PATCH] Improve CI Speed (#398) * use different cache * cache prisma client in rustfmt * cleanup prettier config * build server image in sub-action * server image + prisma client custom actions * remove lefthoook checks that are in ci * fix build-server-image * rename Clippy job * rename build core job * use custom rust caching action * fix rust cache shared key * cleanup custom action job names * remove platform check in clippy ci * setup pnpm in clippy ci * update mobile package name * make ios compiling conditional * use pnpm cache * run clippy in release * force use release for all jobs --- .github/actions/cache-rust-deps/action.yaml | 10 + .../generate-prisma-client/action.yaml | 17 ++ .../actions/publish-server-image/action.yaml | 48 +++++ .github/scripts/setup-system.sh | 1 - .github/workflows/ci.yml | 180 ++++++++---------- .github/workflows/clippy.yml | 58 ------ .prettierignore | 0 .prettierrc.cli.js | 6 - .prettierrc.js | 14 ++ .prettierrc.json | 13 -- Cargo.lock | Bin 152641 -> 152645 bytes .../ios/Spacedrive.xcodeproj/project.pbxproj | 10 +- apps/mobile/rust/Cargo.toml | 3 +- apps/mobile/rust/src/ios.rs | 2 +- lefthook.yml | 14 +- 15 files changed, 172 insertions(+), 204 deletions(-) create mode 100644 .github/actions/cache-rust-deps/action.yaml create mode 100644 .github/actions/generate-prisma-client/action.yaml create mode 100644 .github/actions/publish-server-image/action.yaml delete mode 100644 .github/workflows/clippy.yml delete mode 100644 .prettierignore delete mode 100644 .prettierrc.cli.js create mode 100644 .prettierrc.js delete mode 100644 .prettierrc.json diff --git a/.github/actions/cache-rust-deps/action.yaml b/.github/actions/cache-rust-deps/action.yaml new file mode 100644 index 000000000..ff8bad45d --- /dev/null +++ b/.github/actions/cache-rust-deps/action.yaml @@ -0,0 +1,10 @@ + +name: Cache Rust Dependencies +description: Caches Rust dependencies +runs: + using: 'composite' + steps: + - name: Cache Rust Dependencies + uses: Swatinem/rust-cache@v2 + with: + shared-key: deps-release-v1 diff --git a/.github/actions/generate-prisma-client/action.yaml b/.github/actions/generate-prisma-client/action.yaml new file mode 100644 index 000000000..056afc346 --- /dev/null +++ b/.github/actions/generate-prisma-client/action.yaml @@ -0,0 +1,17 @@ +name: Generate Prisma client +description: Generates the Prisma client, using a cached value if one is available +runs: + using: 'composite' + steps: + - name: Cache Prisma codegen + id: cache-prisma + uses: actions/cache@v3 + with: + path: ./core/src/prisma.rs + key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/*', './Cargo.toml') }} + + - name: Generate Prisma client + working-directory: core + if: steps.cache-prisma.outputs.cache-hit != 'true' + shell: bash + run: cargo run -p prisma-cli --release -- generate diff --git a/.github/actions/publish-server-image/action.yaml b/.github/actions/publish-server-image/action.yaml new file mode 100644 index 000000000..8727e1c34 --- /dev/null +++ b/.github/actions/publish-server-image/action.yaml @@ -0,0 +1,48 @@ +name: Publish server image +description: Builds the server and publishes its Docker image +runs: + using: 'composite' + steps: + - name: Copy binary to Docker context + shell: bash + run: | + cargo build -p server --release + cp ./target/release/server ./apps/server/server + + - name: Determine image name & tag + shell: bash + run: | + if [ "$GITHUB_EVENT_NAME" == "release" ]; then + export IMAGE_TAG=${GITHUB_REF##*/} + else + export IMAGE_TAG=$(git rev-parse --short "$GITHUB_SHA") + fi + export GITHUB_REPOSITORY_LOWER=$(echo $GITHUB_REPOSITORY | awk '{print tolower($0)}') + export IMAGE_NAME="ghcr.io/$GITHUB_REPOSITORY_LOWER/server" + echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV + echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV + echo "Building $IMAGE_NAME:$IMAGE_TAG" + + - name: Build Docker image + shell: bash + run: | + docker build ./apps/server --tag $IMAGE_NAME:$IMAGE_TAG + + - name: Push Docker image + shell: bash + run: | + docker push $IMAGE_NAME:$IMAGE_TAG + + - name: Tag & push image as latest staging image + if: github.event_name != 'release' + shell: bash + run: | + docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:staging + docker push $IMAGE_NAME:staging + + - name: Tag & push image as latest production image + if: github.event_name == 'release' + shell: bash + run: | + docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:production + docker push $IMAGE_NAME:production diff --git a/.github/scripts/setup-system.sh b/.github/scripts/setup-system.sh index 2c8ace653..e968998ac 100755 --- a/.github/scripts/setup-system.sh +++ b/.github/scripts/setup-system.sh @@ -112,5 +112,4 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then else echo "Your OS '$OSTYPE' is not supported by this script. We would welcome a PR or some help adding your OS to this script. https://github.com/spacedriveapp/spacedrive/issues" exit 1 - fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb51550b1..746198d4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,6 @@ on: workflow_dispatch: env: - CARGO_INCREMENTAL: 1 SPACEDRIVE_CUSTOM_APT_FLAGS: --no-install-recommends jobs: @@ -21,36 +20,23 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Install Node.js - uses: actions/setup-node@v3 - with: - node-version: 17 - - name: Install pnpm uses: pnpm/action-setup@v2.2.2 with: version: 7.x.x - run_install: false - - name: Get pnpm store directory - id: pnpm-cache - run: | - echo "::set-output name=pnpm_cache_dir::$(pnpm store path)" - - - uses: actions/cache@v3 - name: Setup pnpm cache + - name: Install Node.js + uses: actions/setup-node@v3 with: - path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + node-version: 17 + cache: 'pnpm' - name: Install pnpm dependencies - run: pnpm --frozen-lockfile i + run: pnpm i --frozen-lockfile - name: Perform typechecks run: pnpm typecheck - + rustfmt: name: rustfmt runs-on: ubuntu-latest @@ -64,11 +50,58 @@ jobs: toolchain: stable components: rustfmt + - name: Cache Rust dependencies + uses: ./.github/actions/cache-rust-deps + + - name: Generate Prisma client + uses: ./.github/actions/generate-prisma-client + - name: Run rustfmt run: cargo fmt --all -- --check - build-core: - name: Build Core (${{ matrix.platform }}) + clippy_check: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + components: clippy + + - name: Cache Rust dependencies + uses: ./.github/actions/cache-rust-deps + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 17 + + - name: Install pnpm + uses: pnpm/action-setup@v2.0.1 + with: + version: 7 + run_install: false + + - name: Run 'setup-system.sh' script + run: ./.github/scripts/setup-system.sh + + - name: Generate Prisma client + uses: ./.github/actions/generate-prisma-client + + - name: Run Clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features --release + + build-and-publish: + name: Build and Publish (${{ matrix.platform }}) runs-on: ${{ matrix.platform }} strategy: fail-fast: true @@ -86,17 +119,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Install Node.js - uses: actions/setup-node@v3 - with: - node-version: 17 - - - name: Install pnpm - uses: pnpm/action-setup@v2.0.1 - with: - version: 7 - run_install: false - - name: Install Rust stable uses: actions-rs/toolchain@v1 with: @@ -105,10 +127,22 @@ jobs: override: true components: rustfmt, rust-src - - name: Cache Rust Dependencies - uses: Swatinem/rust-cache@v1 + - name: Cache Rust dependencies + uses: ./.github/actions/cache-rust-deps + + - name: Install pnpm + uses: pnpm/action-setup@v2.2.2 with: - sharedKey: core-v1-${{ hashFiles('**/Cargo.lock') }} + version: 7.x.x + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 17 + cache: 'pnpm' + + - name: Install pnpm dependencies + run: pnpm i --frozen-lockfile - name: Run 'setup-system.sh' script if: matrix.platform == 'ubuntu-latest' || matrix.platform == 'macos-latest' @@ -118,82 +152,18 @@ jobs: if: matrix.platform == 'windows-latest' run: ./.github/scripts/setup-system.ps1 -ci - - name: Get pnpm store directory - id: pnpm-cache - run: | - echo "::set-output name=pnpm_cache_dir::$(pnpm store path)" - - uses: actions/cache@v3 - name: Setup pnpm cache - with: - path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - name: Install pnpm dependencies - run: pnpm --frozen-lockfile i - - - name: Cache Prisma codegen - id: cache-prisma - uses: actions/cache@v3 - with: - path: ./core/src/prisma.rs - key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/*', './Cargo.toml') }} - - name: Generate Prisma client - working-directory: core - if: steps.cache-prisma.outputs.cache-hit != 'true' - run: cargo run -p prisma-cli --release -- generate + uses: ./.github/actions/generate-prisma-client - - name: Cargo fetch - run: cargo fetch - - - name: Check Core - run: cargo check -p sd-core --release + - name: Build everything + run: cargo build --release - name: Bundle Desktop run: pnpm desktop tauri build - - name: Build Server - if: matrix.platform == 'ubuntu-latest' - run: | - cargo build -p server --release - cp ./target/release/server ./apps/server/server - - - name: Determine image name & tag + - name: Publish server Docker image if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request' - run: | - if [ "$GITHUB_EVENT_NAME" == "release" ]; then - export IMAGE_TAG=${GITHUB_REF##*/} - else - export IMAGE_TAG=$(git rev-parse --short "$GITHUB_SHA") - fi - export GITHUB_REPOSITORY_LOWER=$(echo $GITHUB_REPOSITORY | awk '{print tolower($0)}') - export IMAGE_NAME="ghcr.io/$GITHUB_REPOSITORY_LOWER/server" - echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV - echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV - echo "Building $IMAGE_NAME:$IMAGE_TAG" - - - name: Build Docker image - if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request' - run: | - docker build ./apps/server --tag $IMAGE_NAME:$IMAGE_TAG - - - name: Push Docker image - if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request' - run: | - docker push $IMAGE_NAME:$IMAGE_TAG - - - name: Tag & push image as latest staging image - if: matrix.platform == 'ubuntu-latest' && github.event_name != 'pull_request' && github.event_name != 'release' - run: | - docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:staging - docker push $IMAGE_NAME:staging - - - name: Tag & push image as latest production image - if: matrix.platform == 'ubuntu-latest' && github.event_name == 'release' - run: | - docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:production - docker push $IMAGE_NAME:production + uses: ./.github/actions/publish-server-image - name: Publish artifacts uses: ./.github/actions/publish-artifacts diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml deleted file mode 100644 index 849990b0f..000000000 --- a/.github/workflows/clippy.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Rust Clippy check - -on: - pull_request: - push: - branches: - - main - paths: - - '**.rs' - - '**.toml' - - ./.github/workflows - workflow_dispatch: - -env: - CARGO_INCREMENTAL: 1 - SPACEDRIVE_CUSTOM_APT_FLAGS: --no-install-recommends - SPACEDRIVE_SKIP_PNPM_CHECK: true - -jobs: - clippy_check: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install Rust stable - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true - components: clippy - - - name: Cache Rust Dependencies - uses: Swatinem/rust-cache@v1 - with: - sharedKey: core-v1-${{ hashFiles('**/Cargo.lock') }} - - - name: Run 'setup-system.sh' script - run: ./.github/scripts/setup-system.sh - - - name: Cache Prisma codegen - id: cache-prisma - uses: actions/cache@v3 - with: - path: ./core/src/prisma.rs - key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }} - - - name: Generate Prisma client - working-directory: core - if: steps.cache-prisma.outputs.cache-hit != 'true' - run: cargo run -p prisma-cli --release -- generate - - - name: Run Clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/.prettierrc.cli.js b/.prettierrc.cli.js deleted file mode 100644 index 26d1f83c1..000000000 --- a/.prettierrc.cli.js +++ /dev/null @@ -1,6 +0,0 @@ -var mainConfig = require('./.prettierrc.json'); - -module.exports = { - ...mainConfig, - plugins: ['@trivago/prettier-plugin-sort-imports'] -}; diff --git a/.prettierrc.js b/.prettierrc.js new file mode 100644 index 000000000..f75b3d3d2 --- /dev/null +++ b/.prettierrc.js @@ -0,0 +1,14 @@ +module.exports = { + pluginSearchDirs: ['.'], + useTabs: true, + printWidth: 100, + singleQuote: true, + trailingComma: 'none', + bracketSameLine: false, + semi: true, + quoteProps: 'consistent', + importOrder: ['^@sd/interface/(.*)$', '^@sd/client/(.*)$', '^@sd/ui/(.*)$', '^[./]'], + importOrderSeparation: true, + importOrderSortSpecifiers: true, + plugins: ['@trivago/prettier-plugin-sort-imports'] +}; diff --git a/.prettierrc.json b/.prettierrc.json deleted file mode 100644 index 5347ec663..000000000 --- a/.prettierrc.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "pluginSearchDirs": ["."], - "useTabs": true, - "printWidth": 100, - "singleQuote": true, - "trailingComma": "none", - "bracketSameLine": false, - "semi": true, - "quoteProps": "consistent", - "importOrder": ["^@sd/interface/(.*)$", "^@sd/client/(.*)$", "^@sd/ui/(.*)$", "^[./]"], - "importOrderSeparation": true, - "importOrderSortSpecifiers": true -} diff --git a/Cargo.lock b/Cargo.lock index ea2654f04cb5cc8a578ef1512727a5a67b2d65e4..6c306e8b28b5915f50c7ecdeee4135879f499e4c 100644 GIT binary patch delta 49 zcmV-10M7rxs0qcW34pW#w2um7Z*pZVZEs>}Y?t8|0u`6ej{y^x8<7E}x3Z4`IRUr8 HkpY3i1Ys0v delta 52 zcmV-40L%Zys0qQS34pW#w2!xqkO4OV31nk$a%C-SX=0O6?-ZASvjG#A=8plTm;RCg K5x3Qm0dT@ocoygY diff --git a/apps/mobile/ios/Spacedrive.xcodeproj/project.pbxproj b/apps/mobile/ios/Spacedrive.xcodeproj/project.pbxproj index 4299b55a6..003b6e4b7 100644 --- a/apps/mobile/ios/Spacedrive.xcodeproj/project.pbxproj +++ b/apps/mobile/ios/Spacedrive.xcodeproj/project.pbxproj @@ -266,7 +266,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/zsh; - shellScript = "set -e\n\nif [[ -n \"${DEVELOPER_SDK_DIR:-}\" ]]; then\n # Assume we're in Xcode, which means we're probably cross-compiling.\n # In this case, we need to add an extra library search path for build scripts and proc-macros,\n # which run on the host instead of the target.\n # (macOS Big Sur does not have linkable libraries in /usr/lib/.)\n export LIBRARY_PATH=\"${DEVELOPER_SDK_DIR}/MacOSX.sdk/usr/lib:${LIBRARY_PATH:-}\"\nfi\n\nCARGO_FLAGS=\nif [[ \"$BUILDVARIANT\" != \"debug\" ]]; then\n CARGO_FLAGS=--release\nfi\n\nTARGET_DIRECTORY=../../../target\nif [[ $PLATFORM_NAME = \"iphonesimulator\" ]]\nthen\n cargo build -p sdcore-lib $CARGO_FLAGS --lib --target aarch64-apple-ios-sim\n lipo -create -output $TARGET_DIRECTORY/libsdcore-iossim.a $TARGET_DIRECTORY/aarch64-apple-ios-sim/release/libsdcore.a\nelse\n cargo build -p sdcore-lib $CARGO_FLAGS --lib --target aarch64-apple-ios\n lipo -create -output $TARGET_DIRECTORY/libsdcore-ios.a $TARGET_DIRECTORY/aarch64-apple-ios/release/libsdcore.a\nfi\n"; + shellScript = "set -e\n\nif [[ -n \"${DEVELOPER_SDK_DIR:-}\" ]]; then\n # Assume we're in Xcode, which means we're probably cross-compiling.\n # In this case, we need to add an extra library search path for build scripts and proc-macros,\n # which run on the host instead of the target.\n # (macOS Big Sur does not have linkable libraries in /usr/lib/.)\n export LIBRARY_PATH=\"${DEVELOPER_SDK_DIR}/MacOSX.sdk/usr/lib:${LIBRARY_PATH:-}\"\nfi\n\nCARGO_FLAGS=\nif [[ \"$BUILDVARIANT\" != \"debug\" ]]; then\n CARGO_FLAGS=--release\nfi\n\nTARGET_DIRECTORY=../../../target\nif [[ $PLATFORM_NAME = \"iphonesimulator\" ]]\nthen\n cargo build -p sd-core-mobile $CARGO_FLAGS --lib --target aarch64-apple-ios-sim\n lipo -create -output $TARGET_DIRECTORY/libsd_core_mobile-iossim.a $TARGET_DIRECTORY/aarch64-apple-ios-sim/release/libsd_core_mobile.a\nelse\n cargo build -p sdcore-lib $CARGO_FLAGS --lib --target aarch64-apple-ios\n lipo -create -output $TARGET_DIRECTORY/libsd_core_mobile-ios.a $TARGET_DIRECTORY/aarch64-apple-ios/release/libsd_core_mobile.a\nfi\n"; }; 9F553C6F8AA059AB72DAA720 /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; @@ -420,13 +420,13 @@ "$(inherited)", "-ObjC", "-lc++", - "-lsdcore-ios", + "-lsd_core_mobile-ios", ); "OTHER_LDFLAGS[sdk=iphonesimulator*]" = ( "$(inherited)", "-ObjC", "-lc++", - "-lsdcore-iossim", + "-lsd_core_mobile-iossim", ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG"; PRODUCT_BUNDLE_IDENTIFIER = com.spacedrive.app; @@ -509,13 +509,13 @@ "$(inherited)", "-ObjC", "-lc++", - "-lsdcore-ios", + "-lsd_core_mobile-ios", ); "OTHER_LDFLAGS[sdk=iphonesimulator*]" = ( "$(inherited)", "-ObjC", "-lc++", - "-lsdcore-iossim", + "-lsd_core_mobile-iossim", ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE"; PRODUCT_BUNDLE_IDENTIFIER = com.spacedrive.app; diff --git a/apps/mobile/rust/Cargo.toml b/apps/mobile/rust/Cargo.toml index 955690969..dde522bbe 100644 --- a/apps/mobile/rust/Cargo.toml +++ b/apps/mobile/rust/Cargo.toml @@ -1,11 +1,10 @@ [package] -name = "sdcore-lib" +name = "sd-core-mobile" version = "0.1.0" edition = "2021" rust-version = "1.64.0" [lib] -name = "sdcore" crate-type = ["staticlib", "cdylib"] # staticlib for IOS and cdylib for Android [dependencies] diff --git a/apps/mobile/rust/src/ios.rs b/apps/mobile/rust/src/ios.rs index 3f9c0821f..19d26d8eb 100644 --- a/apps/mobile/rust/src/ios.rs +++ b/apps/mobile/rust/src/ios.rs @@ -70,7 +70,7 @@ pub unsafe extern "C" fn sd_core_msg(query: *const c_char, resolve: *const c_voi .to_str() .unwrap() .to_string(); - let new_node = Node::new(doc_dir).await; + let new_node = Node::new(doc_dir).await.unwrap(); node.replace(new_node.clone()); new_node } diff --git a/lefthook.yml b/lefthook.yml index bc7831af6..d87887b94 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -6,21 +6,9 @@ pre-push: parallel: true commands: - type-check: - glob: '*.{ts,tsx}' - run: pnpm typecheck lint: glob: '*.{ts,tsx}' run: pnpm eslint {staged_files} markdown-link-check: glob: '*.md' - run: pnpm markdown-link-check {staged_files} - rust-fmt: - glob: '*.rs' - run: cargo fmt --all -- --check - rust-lint-tauri: - run: cargo clippy --package spacedrive -- -D warnings - rust-lint-core: - run: cargo clippy --package sd-core --lib -- -D warnings - rust-lint-server: - run: cargo clippy --package server -- -D warnings + run: pnpm markdown-link-check {staged_files} \ No newline at end of file