From 41b2a959eddc60d2a010f82d5c2d1c82d61a7bca Mon Sep 17 00:00:00 2001
From: Leendert de Borst <ldeborst@xivisoft.com>
Date: Mon, 23 Jun 2025 13:59:54 +0200
Subject: [PATCH] Add scripts to convert EF core structure to Typescript
 definitions (#955)

---
 .../AliasClientDb/Scripts/.gitignore          |   3 +
 .../Databases/AliasClientDb/Scripts/README.md |   7 +
 .../Scripts/convert-sql-to-ts.sh              | 233 ++++++++++++++++++
 .../Scripts/generate-sql-files.sh             |  62 +++++
 .../AliasClientDb/Scripts/run-all.sh          |  47 ++++
 docs/misc/dev/upgrade-ef-client-model.md      |  67 ++++-
 6 files changed, 412 insertions(+), 7 deletions(-)
 create mode 100644 apps/server/Databases/AliasClientDb/Scripts/.gitignore
 create mode 100644 apps/server/Databases/AliasClientDb/Scripts/README.md
 create mode 100755 apps/server/Databases/AliasClientDb/Scripts/convert-sql-to-ts.sh
 create mode 100755 apps/server/Databases/AliasClientDb/Scripts/generate-sql-files.sh
 create mode 100755 apps/server/Databases/AliasClientDb/Scripts/run-all.sh

diff --git a/apps/server/Databases/AliasClientDb/Scripts/.gitignore b/apps/server/Databases/AliasClientDb/Scripts/.gitignore
new file mode 100644
index 000000000..980b1323d
--- /dev/null
+++ b/apps/server/Databases/AliasClientDb/Scripts/.gitignore
@@ -0,0 +1,3 @@
+# Ignore generated files
+MigrationSql
+MigrationTs
diff --git a/apps/server/Databases/AliasClientDb/Scripts/README.md b/apps/server/Databases/AliasClientDb/Scripts/README.md
new file mode 100644
index 000000000..bf70d0276
--- /dev/null
+++ b/apps/server/Databases/AliasClientDb/Scripts/README.md
@@ -0,0 +1,7 @@
+# SQL Generation Scripts
+
+This directory contains scripts to generate SQL files from Entity Framework Core migrations and convert them to TypeScript constants which are used by the `./shared/vault-sql` shared TS library.
+
+This shared TS library is consumed by the web app, browser extensions and mobile apps for vault creation and upgrades.
+
+Refer to the docs `upgrade-ef-client-model.md` for how this scripts are used.
\ No newline at end of file
diff --git a/apps/server/Databases/AliasClientDb/Scripts/convert-sql-to-ts.sh b/apps/server/Databases/AliasClientDb/Scripts/convert-sql-to-ts.sh
new file mode 100755
index 000000000..9eb72e55e
--- /dev/null
+++ b/apps/server/Databases/AliasClientDb/Scripts/convert-sql-to-ts.sh
@@ -0,0 +1,233 @@
+#!/bin/bash
+
+# Script to convert SQL migration files to TypeScript constants
+# Run this after generate-sql-files.sh to create SqlConstants.ts and VaultVersions.ts
+
+# Configurable settings
+SQL_DIR="MigrationSql"
+OUTPUT_FILE="MigrationTs/SqlConstants.ts"
+VERSIONS_FILE="MigrationTs/VaultVersions.ts"
+TEMP_DIR="/tmp/sql_to_ts"
+
+# Path to the shared vault-sql package
+SHARED_SQL_DIR="../../../../../shared/vault-sql/src/sql"
+
+# Create temp directory and output directory
+mkdir -p "$TEMP_DIR"
+mkdir -p "$(dirname "$OUTPUT_FILE")"
+
+# Function to escape SQL content for TypeScript
+escape_sql_for_ts() {
+    local sql_content="$1"
+    # Escape backticks and backslashes
+    echo "$sql_content" | sed 's/\\/\\\\/g' | sed 's/`/\\`/g'
+}
+
+# Function to extract migration number from filename
+extract_migration_number() {
+    local filename="$1"
+    # Extract number from filename like "001_InitialMigration_to_AddEmail.sql"
+    echo "$filename" | sed -n 's/^0*\([0-9]*\)_.*\.sql$/\1/p'
+}
+
+# Function to extract migration name from filename
+extract_migration_name() {
+    local filename="$1"
+    # Extract name from filename like "001_InitialMigration_to_AddEmail.sql"
+    echo "$filename" | sed -n 's/^[0-9]*_\(.*\)\.sql$/\1/p'
+}
+
+# Function to extract version, description, and release date from filename (using the last migration segment)
+extract_version_info() {
+    local filename="$1"
+    # Get the last migration segment (after the last _)
+    local last_segment=$(echo "$filename" | awk -F'_to_' '{print $NF}' | sed 's/\.sql$//')
+    # last_segment example: 20250310131554_1.5.0-AddTotpCodes
+    local timestamp=$(echo "$last_segment" | awk -F'_' '{print $1}')
+    local version_and_desc=$(echo "$last_segment" | awk -F'_' '{print $2}')
+    local version=$(echo "$version_and_desc" | awk -F'-' '{print $1}')
+    local description=$(echo "$version_and_desc" | awk -F'-' '{print $2}')
+    # Format date
+    local year=${timestamp:0:4}
+    local month=${timestamp:4:2}
+    local day=${timestamp:6:2}
+    local release_date="$year-$month-$day"
+    # Make description readable
+    local readable_desc=$(echo "$description" | sed 's/\([A-Z]\)/ \1/g' | sed 's/^./\U&/' | sed 's/^U //;s/^ *//')
+    echo "$version|$readable_desc|$release_date"
+}
+
+# Function to copy files to shared vault-sql directory
+copy_to_shared_sql() {
+    local source_file="$1"
+    local target_file="$2"
+
+    if [ -f "$source_file" ]; then
+        echo "Copying $source_file to $target_file"
+        cp "$source_file" "$target_file"
+        if [ $? -eq 0 ]; then
+            echo "✓ Successfully copied to shared vault-sql directory"
+        else
+            echo "✗ Failed to copy to shared vault-sql directory"
+            return 1
+        fi
+    else
+        echo "✗ Source file $source_file not found"
+        return 1
+    fi
+}
+
+echo "Converting SQL files to TypeScript constants..."
+
+# Start building the TypeScript file
+cat > "$OUTPUT_FILE" << 'EOF'
+/* eslint-disable no-irregular-whitespace */
+
+/**
+ * Complete database schema SQL (latest version)
+ * Auto-generated from EF Core migrations
+ */
+export const COMPLETE_SCHEMA_SQL = `
+EOF
+
+# Add the full schema SQL if it exists
+if [ -f "$SQL_DIR/000_FullSchema.sql" ]; then
+    echo "Adding complete schema SQL..."
+    FULL_SCHEMA=$(cat "$SQL_DIR/000_FullSchema.sql")
+    ESCAPED_SCHEMA=$(escape_sql_for_ts "$FULL_SCHEMA")
+    echo "$ESCAPED_SCHEMA" >> "$OUTPUT_FILE"
+else
+    echo "Warning: 000_FullSchema.sql not found"
+fi
+
+# Close the complete schema constant
+cat >> "$OUTPUT_FILE" << 'EOF'
+`;
+/**
+ * Individual migration SQL scripts
+ * Auto-generated from EF Core migrations
+ */
+export const MIGRATION_SCRIPTS: Record<number, string> = {
+EOF
+
+# Process all migration files (excluding the full schema)
+migration_files=($(ls "$SQL_DIR"/*.sql | grep -v "000_FullSchema.sql" | sort))
+
+if [ ${#migration_files[@]} -eq 0 ]; then
+    echo "No migration files found in $SQL_DIR"
+    # Add empty object
+    echo "};" >> "$OUTPUT_FILE"
+    exit 0
+fi
+
+# Process each migration file
+for file in "${migration_files[@]}"; do
+    filename=$(basename "$file")
+    migration_number=$(extract_migration_number "$filename")
+    migration_name=$(extract_migration_name "$filename")
+
+    if [ -z "$migration_number" ]; then
+        echo "Warning: Could not extract migration number from $filename, skipping..."
+        continue
+    fi
+
+    echo "Processing migration $migration_number: $migration_name"
+
+    # Read and escape SQL content
+    sql_content=$(cat "$file")
+    escaped_sql=$(escape_sql_for_ts "$sql_content")
+
+    # Add to TypeScript file
+    cat >> "$OUTPUT_FILE" << EOF
+  $migration_number: \`$escaped_sql\`,
+EOF
+done
+
+# Close the TypeScript file
+cat >> "$OUTPUT_FILE" << 'EOF'
+};
+EOF
+
+echo "TypeScript constants file generated: $OUTPUT_FILE"
+echo "Total migrations processed: ${#migration_files[@]}"
+
+# Now generate the vault versions file
+echo ""
+echo "Generating vault versions mapping..."
+
+# Start building the vault versions file
+cat > "$VERSIONS_FILE" << 'EOF'
+/**
+ * Vault version information
+ * Auto-generated from EF Core migration filenames
+ */
+
+import { IVaultVersion } from "../types/VaultVersion";
+
+/**
+ * Available vault versions in chronological order
+ */
+export const VAULT_VERSIONS: IVaultVersion[] = [
+EOF
+
+# Remove: declare -A versions_seen
+last_version=""
+for file in "${migration_files[@]}"; do
+    filename=$(basename "$file")
+    # Extract revision from filename prefix
+    revision=$(echo "$filename" | sed -n 's/^0*\([0-9]*\)_.*$/\1/p')
+    # Debug: print filename
+    echo "DEBUG: Processing file $filename"
+    last_segment=$(echo "$filename" | awk -F'_to_' '{print $NF}' | sed 's/\.sql$//')
+    echo "DEBUG: Last segment: $last_segment"
+    version_info=$(extract_version_info "$filename")
+    version=$(echo "$version_info" | cut -d'|' -f1)
+    description=$(echo "$version_info" | cut -d'|' -f2)
+    release_date=$(echo "$version_info" | cut -d'|' -f3)
+
+    echo "DEBUG: $filename -> revision='$revision', version='$version', description='$description', release_date='$release_date'"
+
+    # Only output if version is not empty and not a duplicate of the last one
+    if [ -n "$version" ] && [ -n "$description" ] && [ "$version" != "$last_version" ]; then
+        last_version="$version"
+        echo "Found version $version: $description ($release_date)"
+        cat >> "$VERSIONS_FILE" << EOF
+  {
+    revision: $revision,
+    version: '$version',
+    description: '$description',
+    releaseDate: '$release_date'
+  },
+EOF
+    fi
+done
+
+# Close the vault versions file
+cat >> "$VERSIONS_FILE" << 'EOF'
+];
+EOF
+
+echo "Vault versions file generated: $VERSIONS_FILE"
+
+# Copy generated files to shared vault-sql directory
+echo ""
+echo "Copying generated files to shared vault-sql directory..."
+
+# Check if shared directory exists
+if [ ! -d "$SHARED_SQL_DIR" ]; then
+    echo "✗ Shared vault-sql directory not found: $SHARED_SQL_DIR"
+    echo "Make sure you're running this script from the correct location"
+    exit 1
+fi
+
+# Copy SqlConstants.ts
+copy_to_shared_sql "$OUTPUT_FILE" "$SHARED_SQL_DIR/SqlConstants.ts"
+
+# Copy VaultVersions.ts
+copy_to_shared_sql "$VERSIONS_FILE" "$SHARED_SQL_DIR/VaultVersions.ts"
+
+echo ""
+echo "Done!"
+
+# Clean up temp directory
+rm -rf "$TEMP_DIR"
\ No newline at end of file
diff --git a/apps/server/Databases/AliasClientDb/Scripts/generate-sql-files.sh b/apps/server/Databases/AliasClientDb/Scripts/generate-sql-files.sh
new file mode 100755
index 000000000..a7356f614
--- /dev/null
+++ b/apps/server/Databases/AliasClientDb/Scripts/generate-sql-files.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Make sure to install the dotnet ef tool first before running this script:
+# dotnet tool install --global dotnet-ef
+
+# Configurable settings
+PROJECT="../AliasClientDb.csproj"
+STARTUP_PROJECT="../AliasClientDb.csproj"   # Adjust if different from main project
+CONTEXT="AliasClientDbContext"
+OUTPUT_DIR="MigrationSql"
+FULL_FILE="$OUTPUT_DIR/000_FullSchema.sql"
+
+# Create output directory
+mkdir -p "$OUTPUT_DIR"
+
+# Use 'set --' to build a list of pending migrations into $1 $2 ...
+set -- $(dotnet ef migrations list \
+  --project "$PROJECT" \
+  --startup-project "$STARTUP_PROJECT" \
+  --context "$CONTEXT" 2>/dev/null \
+  | grep '(Pending)' \
+  | sed 's/ (Pending)//')
+
+TOTAL=$#
+
+# Generate full script from scratch if any pending migrations exist
+if [ "$TOTAL" -gt 0 ]; then
+  echo "Generating full schema script..."
+  dotnet ef migrations script \
+    --project "$PROJECT" \
+    --startup-project "$STARTUP_PROJECT" \
+    --context "$CONTEXT" \
+    --output "$FULL_FILE"
+else
+  echo "No pending migrations found. Skipping full script."
+fi
+
+# Also generate per-migration scripts if enough exist
+if [ "$TOTAL" -lt 2 ]; then
+  echo "Not enough pending migrations to generate step-by-step scripts."
+  exit 0
+fi
+
+# Loop over migration pairs
+i=1
+while [ "$i" -le "$TOTAL" ]; do
+  j=$((i + 1))
+  FROM=$(eval echo \${$i})
+  TO=$(eval echo \${$j})
+  FILE="$OUTPUT_DIR/$(printf "%03d" "$i")_${FROM}_to_${TO}.sql"
+
+  echo "Generating script: $FROM -> $TO"
+  dotnet ef migrations script "$FROM" "$TO" \
+    --project "$PROJECT" \
+    --startup-project "$STARTUP_PROJECT" \
+    --context "$CONTEXT" \
+    --output "$FILE"
+
+  i=$((i + 1))
+done
+
+echo "Done. Scripts written to $OUTPUT_DIR"
\ No newline at end of file
diff --git a/apps/server/Databases/AliasClientDb/Scripts/run-all.sh b/apps/server/Databases/AliasClientDb/Scripts/run-all.sh
new file mode 100755
index 000000000..c4cb71c72
--- /dev/null
+++ b/apps/server/Databases/AliasClientDb/Scripts/run-all.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Combined script to generate SQL files and TypeScript constants
+# This script runs both generate-sql-files.sh and generate-sql-constants.sh
+
+echo "=== AliasVault SQL Generation Pipeline ==="
+echo ""
+
+# Step 1: Generate SQL files from EF Core migrations
+echo "Step 1: Generating SQL files from EF Core migrations..."
+if [ -f "generate-sql-files.sh" ]; then
+    bash generate-sql-files.sh
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to generate SQL files"
+        exit 1
+    fi
+else
+    echo "Error: generate-sql-files.sh not found"
+    exit 1
+fi
+
+echo ""
+
+# Step 2: Convert SQL files to TypeScript constants
+echo "Step 2: Converting SQL files to TypeScript constants..."
+if [ -f "convert-sql-to-ts.sh" ]; then
+    bash convert-sql-to-ts.sh
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to generate TypeScript constants"
+        exit 1
+    fi
+else
+    echo "Error: generate-sql-constants.sh not found"
+    exit 1
+fi
+
+echo ""
+echo "=== Pipeline completed successfully! ==="
+echo ""
+echo "Generated files:"
+echo "- SQL files: MigrationSql/"
+echo "- TypeScript files: MigrationTs/"
+echo ""
+echo "The TypeScript files have been copied to the shared vault-sql directory."
+echo "Make sure to rebuild the vault-sql library and test it in the client apps."
+echo ""
+echo "shared/build-and-distribute.sh"
\ No newline at end of file
diff --git a/docs/misc/dev/upgrade-ef-client-model.md b/docs/misc/dev/upgrade-ef-client-model.md
index 714a3c719..2e8221d3e 100644
--- a/docs/misc/dev/upgrade-ef-client-model.md
+++ b/docs/misc/dev/upgrade-ef-client-model.md
@@ -8,15 +8,68 @@ nav_order: 5
 
 # Upgrade the AliasClientDb EF model
 
-To upgrade the AliasClientDb EF model, follow these steps:
+This guide explains how to upgrade the AliasVault client database structure. The AliasVault client database is built and managed using Entity Framework code-first approach, where all SQL structure is maintained in code and then converted to SQL scripts for use across web apps, browser extensions, and mobile apps.
 
-1. Make changes to the AliasClientDb EF model in the `AliasClientDb` project.
-2. Create a new migration by running the following command in the `AliasClientDb` project:
+## Overview
+
+The upgrade process involves three main steps:
+
+1. **Update .NET Entity Framework model** - Modify the EF model and create migrations
+2. **Generate SQL scripts** - Convert EF migrations to SQL scripts for cross-platform use
+3. **Rebuild vault-sql shared library** - Compile and distribute the updated SQL scripts
+
+---
+
+## 1. Update .NET Entity Framework Model
+
+### Step 1.1: Modify the EF Model
+Make changes to the AliasClientDb EF model in the `AliasClientDb` project.
+
+### Step 1.2: Create a New Migration
+Run the following command in the `AliasClientDb` project:
 
 ```bash
-# Important: make sure the migration name is prefixed by the Semver version number of the release.
-# For example, if the release version is 1.0.0, the migration name should be `1.0.0-<migration-name>`.
+# Important: Migration name must be prefixed with the Semver version number of the release.
+# Example: If the release version is 1.0.0, use `1.0.0-<migration-name>`
 dotnet ef migrations add "1.0.0-<migration-name>"
 ```
-4. On the next login of a user, they will be prompted (required) to upgrade their database schema to the latest version.
-Make sure to manually test that the migration works as expected.
+
+**Note:** Always prefix migration names with the release version number to ensure proper versioning across all client platforms.
+
+---
+
+## 2. Generate SQL Scripts
+
+### Step 2.1: Run the SQL Generation Script
+Execute the SQL generation script to convert EF migrations to SQL scripts:
+
+```bash
+apps/server/Databases/AliasClientDb/Scripts/run-all.sh
+```
+
+### Step 2.2: Verify Output
+The script will:
+- Create individual SQL scripts for each migration
+- Convert these to TypeScript versions
+- Save the results in `shared/vault-sql/src/sql` directory
+
+---
+
+## 3. Rebuild vault-sql Shared Library
+
+### Step 3.1: Compile and Distribute
+The vault-sql TypeScript library is consumed by web apps, browser extensions, and mobile apps for vault creation and updates. After generating the SQL scripts, rebuild the library:
+
+```bash
+shared/build-and-distribute.sh
+```
+
+### Step 3.2: Verify Distribution
+Ensure the updated library is properly distributed to all consuming applications.
+
+---
+
+## Testing and Deployment
+
+### Manual Testing
+On the next login of any client app, users will be prompted (required) to upgrade their database schema to the latest version. **Always manually test that the migration works as expected** before releasing.