From 79c8a61c213f0bc20f7e973288dc4c26e48a8223 Mon Sep 17 00:00:00 2001 From: ACX <8075870+acx10@users.noreply.github.com> Date: Sun, 14 Dec 2025 01:57:32 -0700 Subject: [PATCH] Add Flyway migration version conflict check in GitHub Actions (#1863) Co-authored-by: acx10 --- .github/scripts/analyze-changes.sh | 70 +++++++++++++++ .github/scripts/check-conflicts.sh | 96 +++++++++++++++++++++ .github/scripts/determine-compare-ref.sh | 41 +++++++++ .github/scripts/validate-versions.sh | 79 +++++++++++++++++ .github/workflows/docker-build-publish.yml | 99 +++++++++++++--------- 5 files changed, 346 insertions(+), 39 deletions(-) create mode 100644 .github/scripts/analyze-changes.sh create mode 100644 .github/scripts/check-conflicts.sh create mode 100644 .github/scripts/determine-compare-ref.sh create mode 100644 .github/scripts/validate-versions.sh diff --git a/.github/scripts/analyze-changes.sh b/.github/scripts/analyze-changes.sh new file mode 100644 index 00000000..1665f224 --- /dev/null +++ b/.github/scripts/analyze-changes.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Exit immediately if a command exits with a non-zero status. +set -e +# Exit if any command in a pipeline fails, not just the last one. +set -o pipefail + +# Define the path where Flyway migration files are stored. +MIGRATION_PATH="booklore-api/src/main/resources/db/migration" + +# Get ALL changes: Added (A), Modified (M), Renamed (R), Copied (C), Deleted (D) +# for SQL files in the migration path between the comparison ref and the current HEAD. +# The output is saved to a temporary file for further processing. +git diff --name-status --diff-filter=AMRCD $COMPARE_REF...HEAD -- "$MIGRATION_PATH/V*.sql" > /tmp/all_changes.txt + +# The check for no changes is now handled in the workflow. +# If this script runs, it's because changes were detected. + +echo "📝 Migration changes detected:" +# Display the detected changes, indented for readability. +cat /tmp/all_changes.txt | sed 's/^/ /' +echo "" + +# Check for deleted files +# Grep for lines starting with 'D' (Deleted). The '|| true' prevents the script from exiting if no matches are found. +DELETED=$(grep "^D" /tmp/all_changes.txt || true) +if [ -n "$DELETED" ]; then + echo "❌ ERROR: Deleted migration files detected!" + echo "$DELETED" | sed 's/^/ /' + echo "" + echo "Flyway migrations should NEVER be deleted after being applied." + echo "If you need to revert changes, create a new migration." + exit 1 +fi + +# Check for renamed files +# Grep for lines starting with 'R' (Renamed). +RENAMED=$(grep "^R" /tmp/all_changes.txt || true) +if [ -n "$RENAMED" ]; then + echo "❌ ERROR: Renamed migration files detected!" + echo "$RENAMED" | sed 's/^/ /' + echo "" + echo "Flyway migrations should NEVER be renamed after being applied." + echo "This will cause issues with migration history tracking." + echo "" + echo "💡 To fix: Revert the rename and create a new migration file instead." + exit 1 +fi + +# Check for modified files +# Grep for lines starting with 'M' (Modified). +MODIFIED=$(grep "^M" /tmp/all_changes.txt || true) +if [ -n "$MODIFIED" ]; then + echo "❌ ERROR: Modified migration files detected!" + echo "$MODIFIED" | sed 's/^/ /' + echo "" + echo "Flyway migrations should NEVER be modified after being applied." + echo "This will cause checksum validation failures in environments where it has already been applied." + echo "" + echo "💡 To fix: Revert the changes and create a new migration file instead." + exit 1 +fi + +# Extract ADDED files for conflict checking in a later step. +# We grep for lines starting with 'A' (Added), then use 'cut' to get just the file path. +# 'touch' ensures the file exists even if there are no added files. +grep "^A" /tmp/all_changes.txt | cut -f2- > /tmp/pr_files.txt || touch /tmp/pr_files.txt + +# Set a GitHub Actions output variable to indicate that migration changes were found. +# This is used by the workflow to decide whether to run subsequent steps. +echo "has_changes=true" >> $GITHUB_OUTPUT diff --git a/.github/scripts/check-conflicts.sh b/.github/scripts/check-conflicts.sh new file mode 100644 index 00000000..34c843d5 --- /dev/null +++ b/.github/scripts/check-conflicts.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Exit immediately if a command exits with a non-zero status. +set -e +# Exit if any command in a pipeline fails, not just the last one. +set -o pipefail + +# If there are no new versions to check, exit gracefully. +# This file is created by 'validate-versions.sh'. +# This can happen if a PR has changes, but none are new migration files. +if [ ! -s /tmp/versions_pr_unique.txt ]; then + echo "ℹī¸ No new migration versions to check for conflicts." + exit 0 +fi + +# Define the path where Flyway migration files are stored. +MIGRATION_PATH="booklore-api/src/main/resources/db/migration" + +echo "🔍 Fetching migration files from $COMPARE_REF..." + +# Get ALL existing migration files from the comparison ref (e.g., 'develop' or a tag). +# 'git ls-tree' lists the contents of a tree object. +# The output is piped to grep to filter for only Flyway SQL files. +# '|| touch' ensures the temp file exists even if no files are found. +git ls-tree -r --name-only $COMPARE_REF -- "$MIGRATION_PATH/" 2>/dev/null | \ + grep "V.*\.sql$" > /tmp/base_files.txt || touch /tmp/base_files.txt + +# Handle the case where no migration files exist in the base branch. +if [ ! -s /tmp/base_files.txt ]; then + echo "⚠ī¸ No migration files found in $COMPARE_REF" + echo "This might be the first migration or the path has changed." + echo "" + echo "✅ Skipping version conflict check." + + PR_COUNT=$(wc -l < /tmp/versions_pr_unique.txt) + echo "" + echo "📊 Migration Summary:" + echo " - Existing migrations in $COMPARE_REF: 0" + echo " - New migrations in this PR: $PR_COUNT" + exit 0 +fi + +echo "📋 Found $(wc -l < /tmp/base_files.txt) migration files in $COMPARE_REF" + +# Extract versions from the base files. +# The loop reads each file path, extracts the version number from the filename, +# and appends it to a temporary file. +> /tmp/versions_base.txt +while IFS= read -r file; do + filename=$(basename "$file") + # sed extracts the version number (e.g., 1.0.0) from a filename like 'V1.0.0__description.sql'. + version=$(echo "$filename" | sed -n 's/^V\([0-9.]*\)__.*/\1/p') + [ -n "$version" ] && echo "$version" >> /tmp/versions_base.txt +done < /tmp/base_files.txt + +# Create a file with only unique, sorted version numbers from the base. +sort -u /tmp/versions_base.txt > /tmp/versions_base_unique.txt + +BASE_COUNT=$(wc -l < /tmp/versions_base_unique.txt) +echo "📊 Found $BASE_COUNT unique versions in $COMPARE_REF" + +# Find conflicts between base versions and versions from NEW PR files. +# 'comm -12' finds lines common to both sorted files. +CONFLICTS=$(comm -12 /tmp/versions_base_unique.txt /tmp/versions_pr_unique.txt) + +# If conflicts are found, report them and exit with an error. +if [ -n "$CONFLICTS" ]; then + echo "❌ Version conflicts detected!" + echo "" + echo "The following versions from your new migration files already exist in $COMPARE_REF:" + echo "$CONFLICTS" | sed 's/^/ V/' + echo "" + + # Show which files have conflicting versions for easier debugging. + echo "Conflicting files:" + while IFS= read -r version; do + echo " Version V$version exists in:" + grep "V${version}__" /tmp/base_files.txt | xargs -n1 basename | sed 's/^/ BASE: /' + # /tmp/pr_files.txt contains only added files from the PR (from analyze-changes.sh). + grep "V${version}__" /tmp/pr_files.txt | xargs -n1 basename | sed 's/^/ PR: /' + done <<< "$CONFLICTS" + + echo "" + echo "💡 To fix: Use a version number that doesn't exist in $COMPARE_REF" + exit 1 +fi + +echo "✅ No version conflicts detected." + +# Get the count of new migrations in the PR. +PR_COUNT=$(wc -l < /tmp/versions_pr_unique.txt) + +# Print a final summary. +echo "" +echo "📊 Migration Summary:" +echo " - Existing migrations in $COMPARE_REF: $BASE_COUNT" +echo " - New migrations in this PR: $PR_COUNT" diff --git a/.github/scripts/determine-compare-ref.sh b/.github/scripts/determine-compare-ref.sh new file mode 100644 index 00000000..13f8839e --- /dev/null +++ b/.github/scripts/determine-compare-ref.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Exit immediately if a command exits with a non-zero status. +set -e +# Exit if any command in a pipeline fails, not just the last one. +set -o pipefail + +# The target branch of the pull request (e.g., 'develop', 'master') is passed as the first argument. +TARGET_BRANCH="$1" +echo "đŸŽ¯ Target branch: $TARGET_BRANCH" + +# Handle cases where the target branch is not specified, such as a direct push to a branch. +if [ -z "$TARGET_BRANCH" ]; then + echo "⚠ī¸ No target branch specified (e.g., a direct push event). Defaulting to compare with 'develop'." + TARGET_BRANCH="develop" +fi + +# Logic to determine the comparison reference based on the target branch. +if [ "$TARGET_BRANCH" = "master" ]; then + # For PRs to 'master', we compare against the latest git tag. + # This is common for release workflows where 'master' only contains tagged releases. + if ! LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null); then + echo "⚠ī¸ No tags found in repository. Skipping conflict check." + # Set output to signal the workflow to stop. + echo "has_ref=false" >> $GITHUB_OUTPUT + exit 0 + fi + echo "📌 Comparing against last tag: $LAST_TAG" + # Set the COMPARE_REF environment variable for subsequent steps in the job. + echo "COMPARE_REF=$LAST_TAG" >> $GITHUB_ENV +else + # For all other cases (PRs to 'develop', other feature branches, or direct pushes), + # we compare against the 'develop' branch. + echo "🔄 Comparing against head of develop branch" + # Ensure the local 'develop' branch is up-to-date with the remote. + git fetch origin develop:develop + # Set the COMPARE_REF to the remote develop branch. + echo "COMPARE_REF=origin/develop" >> $GITHUB_ENV +fi + +# Set a GitHub Actions output variable to indicate that a valid comparison ref was found. +echo "has_ref=true" >> $GITHUB_OUTPUT diff --git a/.github/scripts/validate-versions.sh b/.github/scripts/validate-versions.sh new file mode 100644 index 00000000..f5396811 --- /dev/null +++ b/.github/scripts/validate-versions.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Exit immediately if a command exits with a non-zero status. +set -e +# Exit if any command in a pipeline fails, not just the last one. +set -o pipefail + +# Define the path where Flyway migration files are stored. +MIGRATION_PATH="booklore-api/src/main/resources/db/migration" + +# --- Part 1: Check for duplicate versions within the PR branch itself --- + +# Get ALL migration files in the current HEAD of the PR branch for an internal duplicate check. +find "$MIGRATION_PATH" -type f -name "V*.sql" > /tmp/all_pr_files.txt + +# Check for duplicate versions within the PR branch. This prevents merging a branch +# that contains multiple files with the same version number. +echo "🔎 Checking for duplicate versions in the branch..." +> /tmp/versions_all_pr.txt +# Loop through all found migration files and extract their version numbers. +while IFS= read -r file; do + filename=$(basename "$file") + # sed extracts the version number (e.g., 1.0.0) from a filename like 'V1.0.0__description.sql'. + version=$(echo "$filename" | sed -n 's/^V\([0-9.]*\)__.*/\1/p') + [ -n "$version" ] && echo "$version" >> /tmp/versions_all_pr.txt +done < /tmp/all_pr_files.txt + +# 'uniq -d' filters for lines that appear more than once in the sorted list. +sort /tmp/versions_all_pr.txt | uniq -d > /tmp/duplicates_in_pr.txt + +# If the duplicates file is not empty, report the error and exit. +if [ -s /tmp/duplicates_in_pr.txt ]; then + echo "❌ Duplicate migration versions found within the branch!" + echo "" + echo "The following versions are duplicated:" + while IFS= read -r version; do + echo " - Version V$version is used by:" + # Show the conflicting files for easy debugging. + grep "V${version}__" /tmp/all_pr_files.txt | xargs -n1 basename | sed 's/^/ /' + done < /tmp/duplicates_in_pr.txt + echo "" + echo "💡 To fix: Ensure all migration files have a unique version number." + exit 1 +fi + +echo "✅ No duplicate versions found within the branch." + +# --- Part 2: Extract versions from NEWLY ADDED files for conflict checking against the base branch --- + +# /tmp/pr_files.txt is created by analyze-changes.sh and contains only ADDED files. +# If the file doesn't exist or is empty, there's nothing to check. +if [ ! -f /tmp/pr_files.txt ] || [ ! -s /tmp/pr_files.txt ]; then + echo "ℹī¸ No new migration files to check for conflicts." + # Set output to signal the workflow to skip the conflict check step. + echo "has_versions=false" >> $GITHUB_OUTPUT + exit 0 +fi + +echo "🔎 Extracting versions from new files..." +> /tmp/versions_pr.txt +# Loop through only the NEWLY ADDED files and extract their versions. +while IFS= read -r file; do + filename=$(basename "$file") + version=$(echo "$filename" | sed -n 's/^V\([0-9.]*\)__.*/\1/p') + [ -n "$version" ] && echo "$version" >> /tmp/versions_pr.txt +done < /tmp/pr_files.txt + +# If no valid versions were extracted from the new files, exit. +if [ ! -s /tmp/versions_pr.txt ]; then + echo "ℹī¸ No versions found in new migration files." + echo "has_versions=false" >> $GITHUB_OUTPUT + exit 0 +fi + +# Create a sorted, unique list of versions from the new files. +# This file will be used by 'check-conflicts.sh'. +sort -u /tmp/versions_pr.txt > /tmp/versions_pr_unique.txt + +# Set output to signal that there are new versions to check for conflicts. +echo "has_versions=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index b046d6b9..5f660dde 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -2,11 +2,70 @@ name: Build, Tag, Push, and Release to GitHub Container Registry on: push: + branches: + - 'master' + - 'develop' + pull_request: branches: - '**' jobs: + flyway-conflict-check: + runs-on: ubuntu-latest + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine comparison reference + id: compare_ref + run: | + bash .github/scripts/determine-compare-ref.sh "${{ github.base_ref }}" + + - name: Check for migration changes + id: check_migrations + if: steps.compare_ref.outputs.has_ref == 'true' + run: | + # Check for changes in the migration directory + if git diff --name-only --diff-filter=AMRCD ${{ env.COMPARE_REF }}...HEAD | grep -q "booklore-api/src/main/resources/db/migration/"; then + echo "Migration file changes detected." + echo "has_migrations=true" >> $GITHUB_OUTPUT + else + echo "No migration file changes detected. Skipping conflict check." + echo "has_migrations=false" >> $GITHUB_OUTPUT + fi + + - name: Analyze migration changes + if: steps.check_migrations.outputs.has_migrations == 'true' + id: analyze_changes + env: + COMPARE_REF: ${{ env.COMPARE_REF }} + run: | + bash .github/scripts/analyze-changes.sh + + - name: Extract and validate versions + if: steps.check_migrations.outputs.has_migrations == 'true' + id: validate_versions + run: | + bash .github/scripts/validate-versions.sh + + - name: Check for version conflicts with base + if: steps.validate_versions.outputs.has_versions == 'true' + env: + COMPARE_REF: ${{ env.COMPARE_REF }} + run: | + bash .github/scripts/check-conflicts.sh + + - name: Cleanup + if: always() && steps.check_migrations.outputs.has_migrations == 'true' + run: | + rm -f /tmp/all_changes.txt /tmp/pr_files.txt /tmp/base_files.txt + rm -f /tmp/versions_*.txt /tmp/duplicates_in_pr.txt + build-and-push: + needs: flyway-conflict-check + if: success() || (needs.flyway-conflict-check.result == 'skipped') runs-on: ubuntu-latest permissions: @@ -196,42 +255,4 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - gh release edit ${{ env.new_tag }} --draft=false - - - name: Notify Discord of New Release - if: false - continue-on-error: true - shell: bash - env: - GH_TOKEN: ${{ github.token }} - DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} - NEW_TAG: ${{ env.new_tag }} - run: | - set -euo pipefail - if [ -z "${DISCORD_WEBHOOK_URL:-}" ]; then - echo "DISCORD_WEBHOOK_URL not set, skipping Discord notification." - exit 0 - fi - release_json=$(gh release view "$NEW_TAG" --json name,body,url) - release_name=$(jq -r '.name' <<< "$release_json") - release_body=$(jq -r '.body' <<< "$release_json") - release_url=$(jq -r '.url' <<< "$release_json") - clean_body=$(echo "$release_body" | tr -d '\r') - max_length=1800 - if [ ${#clean_body} -gt $max_length ]; then - clean_body="${clean_body:0:$((max_length-12))}â€Ļ [truncated]" - fi - payload=$(jq -n \ - --arg title "New Release: $release_name" \ - --arg url "$release_url" \ - --arg desc "$clean_body" \ - '{ - content: null, - embeds: [{ - title: $title, - url: $url, - description: $desc, - color: 3066993 - }] - }') - curl -H "Content-Type: application/json" -d "$payload" "$DISCORD_WEBHOOK_URL" + gh release edit ${{ env.new_tag }} --draft=true \ No newline at end of file