chore: updated rules based logic for migration check , added regex pattern for changes migration parsing, improved step failure detection for easier POF detection

Author: itsharshvb

…ub/oasdiff/.oasdiff-severity-levels.yaml …patibility/.oasdiff-severity-levels.yaml.github/oasdiff/.oasdiff-severity-levels.yaml renamed to .github/api-migration-compatibility/.oasdiff-severity-levels.yaml .github/oasdiff/.oasdiff-severity-levels.yaml renamed to .github/api-migration-compatibility/.oasdiff-severity-levels.yaml

@@ -0,0 +1,108 @@
+# Migration Rules Configuration
+# This file defines SQL patterns that are considered breaking changes or warnings
+# for database migrations, with folder-specific rule application.
+
+# Define reusable rule patterns
+rule_patterns:
+  drop_table:
+    pattern: 'DROP\s+TABLE\s+'
+    description: "Dropping a table removes data and breaks existing queries"
+
+  drop_column:
+    pattern: 'DROP\s+COLUMN\s+'
+    description: "Dropping a column breaks applications expecting it"
+
+  delete_data:
+    pattern: 'DELETE\s+FROM\s+'
+    description: "Deleting data can break application logic"
+
+  truncate_table:
+    pattern: 'TRUNCATE\s+'
+    description: "Truncating removes all data from a table"
+
+  rename_table:
+    pattern: 'RENAME\s+TO\s+'
+    description: "Renaming a table breaks existing queries and code references"
+
+  rename_column:
+    pattern: 'RENAME\s+COLUMN\s+'
+    description: "Renaming a column breaks applications using the old name"
+
+  set_not_null:
+    pattern: 'ALTER\s+COLUMN\s+.*\s+SET\s+NOT\s+NULL'
+    description: "Making an existing nullable column NOT NULL can fail with existing NULL data"
+
+  add_column_not_null_no_default:
+    pattern: 'ADD\s+COLUMN\s+(?!.*DEFAULT).*NOT\s+NULL'
+    description: "Adding a NOT NULL column without a DEFAULT will fail on existing rows"
+
+  alter_column_type:
+    pattern: '(ALTER\s+COLUMN\s+.*\s+TYPE|ALTER\s+COLUMN\s+.*\s+SET\s+DATA\s+TYPE)'
+    description: "Changing column type may cause data loss or conversion issues"
+
+  drop_index:
+    pattern: 'DROP\s+INDEX\s+'
+    description: "Dropping an index can significantly impact query performance"
+
+  drop_constraint:
+    pattern: 'DROP\s+CONSTRAINT\s+'
+    description: "Dropping a constraint may allow invalid data"
+
+  alter_default:
+    pattern: '(SET\s+DEFAULT|DROP\s+DEFAULT)'
+    description: "Changing defaults affects new rows only, may cause inconsistency"
+
+# Apply rules per migration folder
+# Each folder can have its own set of breaking and warning rules
+folder_rules:
+  # V1 migrations
+  migrations:
+    breaking:
+      - drop_table
+      - drop_column
+      - delete_data
+      - truncate_table
+      - rename_table
+      - rename_column
+      - set_not_null
+      - add_column_not_null_no_default
+    warnings:
+      - alter_column_type
+      - drop_index
+      - drop_constraint
+      - alter_default
+
+  # V2 migrations
+  v2_migrations:
+    breaking:
+      - drop_table
+      - drop_column
+      - delete_data
+      - truncate_table
+      - rename_table
+      - rename_column
+      - set_not_null
+      - add_column_not_null_no_default
+    warnings:
+      - alter_column_type
+      - drop_index
+      - drop_constraint
+      - alter_default
+
+  # V2 compatible migrations
+  v2_compatible_migrations:
+    breaking:
+      - drop_table
+      - drop_column
+      - delete_data
+      - truncate_table
+      - rename_table
+      - rename_column
+      - set_not_null
+      - add_column_not_null_no_default
+    warnings:
+      - alter_column_type
+      - drop_index
+      - drop_constraint
+      - alter_default
+

.github/oasdiff/.oasdiff-warn-ignore.yaml …-compatibility/.oasdiff-warn-ignore.yaml.github/oasdiff/.oasdiff-warn-ignore.yaml renamed to .github/api-migration-compatibility/.oasdiff-warn-ignore.yaml .github/oasdiff/.oasdiff-warn-ignore.yaml renamed to .github/api-migration-compatibility/.oasdiff-warn-ignore.yaml

@@ -27,7 +27,7 @@ concurrency:
 
 jobs:
   api-compatibility-check:
-    name: API & Migration Compatibility Check
+    name: API Compatibility Check
     runs-on: ubuntu-latest
 
     steps:
@@ -65,7 +65,7 @@ jobs:
 
           if oasdiff breaking \
             --fail-on ERR \
-            --warn-ignore .github/oasdiff/.oasdiff-warn-ignore.yaml \
+            --warn-ignore .github/api-migration-compatibility/.oasdiff-warn-ignore.yaml \
             base-v1-schema.json pr-v1-schema.json > v1-breaking-report.txt 2>&1; then
             echo "V1 API is backward compatible" > v1-breaking-status.txt
           else
@@ -76,7 +76,7 @@ jobs:
 
           if oasdiff breaking \
             --fail-on ERR \
-            --warn-ignore .github/oasdiff/.oasdiff-warn-ignore.yaml \
+            --warn-ignore .github/api-migration-compatibility/.oasdiff-warn-ignore.yaml \
             base-v2-schema.json pr-v2-schema.json > v2-breaking-report.txt 2>&1; then
             echo "V2 API is backward compatible" > v2-breaking-status.txt
           else
@@ -139,39 +139,65 @@ jobs:
         id: migration_check
         run: |
           BASE_REF="origin/${{ github.event.pull_request.base.ref }}"
-          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "migrations/**/*.sql" 2>/dev/null || echo "")
+          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "migrations/**/up.sql" 2>/dev/null || echo "")
 
           if [[ -z "$NEW_MIGRATIONS" ]]; then
             echo "breaking_changes=0" >> $GITHUB_OUTPUT
             echo "warnings=0" >> $GITHUB_OUTPUT
             exit 0
           fi
 
+          # Load migration rules using yq
+          RULES_FILE=".github/api-migration-compatibility/migration-rules.yaml"
+
+          # Extract breaking rule patterns for migrations folder
+          BREAKING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.migrations.breaking[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
+          # Extract warning rule patterns for migrations folder
+          WARNING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.migrations.warnings[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
           BREAKING=0
           WARNINGS=0
 
           while IFS= read -r file; do
             [[ -z "$file" ]] && continue
-            CONTENT=$(cat "$file")
 
-            if echo "$CONTENT" | grep -iE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" >/dev/null; then
+            # Check for breaking changes
+            if grep -iEq "$BREAKING_PATTERNS" "$file"; then
               echo "BREAKING: $file"
-              echo "$CONTENT" | grep -inE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" | sed 's/^/  /'
-              BREAKING=$((BREAKING + 1))
+              MATCH_COUNT=$(grep -icE "$BREAKING_PATTERNS" "$file")
+              grep -inE "$BREAKING_PATTERNS" "$file" | sed 's/^/  /' || true
+              BREAKING=$((BREAKING + MATCH_COUNT))
             fi
 
-            if echo "$CONTENT" | grep -iE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" >/dev/null; then
+            # Check for warnings
+            if grep -iEq "$WARNING_PATTERNS" "$file"; then
               echo "WARNING: $file"
-              echo "$CONTENT" | grep -inE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" | sed 's/^/  /'
-              WARNINGS=$((WARNINGS + 1))
+              MATCH_COUNT=$(grep -icE "$WARNING_PATTERNS" "$file")
+              grep -inE "$WARNING_PATTERNS" "$file" | sed 's/^/  /' || true
+              WARNINGS=$((WARNINGS + MATCH_COUNT))
             fi
           done <<< "$NEW_MIGRATIONS"
 
           echo "breaking_changes=$BREAKING" >> $GITHUB_OUTPUT
           echo "warnings=$WARNINGS" >> $GITHUB_OUTPUT
-        continue-on-error: true
+
+          if [[ $BREAKING -gt 0 ]]; then
+            echo "::error::Breaking changes detected in V1 database migrations."
+            echo "::error::Found $BREAKING breaking change(s)."
+            exit 1
+          fi
 
       - name: Validate V2 migrations
+        if: success() || failure()
         id: v2_migration_check
         run: |
           if [[ ! -d "v2_migrations" ]]; then
@@ -181,38 +207,65 @@ jobs:
           fi
 
           BASE_REF="origin/${{ github.event.pull_request.base.ref }}"
-          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "v2_migrations/**/*.sql" 2>/dev/null || echo "")
+          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "v2_migrations/**/up.sql" 2>/dev/null || echo "")
 
           if [[ -z "$NEW_MIGRATIONS" ]]; then
             echo "breaking_changes=0" >> $GITHUB_OUTPUT
             echo "warnings=0" >> $GITHUB_OUTPUT
             exit 0
           fi
 
+          # Load migration rules using yq
+          RULES_FILE=".github/api-migration-compatibility/migration-rules.yaml"
+
+          # Extract breaking rule patterns for v2_migrations folder
+          BREAKING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.v2_migrations.breaking[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
+          # Extract warning rule patterns for v2_migrations folder
+          WARNING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.v2_migrations.warnings[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
           BREAKING=0
           WARNINGS=0
 
           while IFS= read -r file; do
             [[ -z "$file" ]] && continue
-            CONTENT=$(cat "$file")
 
-            if echo "$CONTENT" | grep -iE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" >/dev/null; then
+            # Check for breaking changes
+            if grep -iEq "$BREAKING_PATTERNS" "$file"; then
               echo "BREAKING: $file"
-              echo "$CONTENT" | grep -inE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" | sed 's/^/  /'
-              BREAKING=$((BREAKING + 1))
+              MATCH_COUNT=$(grep -icE "$BREAKING_PATTERNS" "$file")
+              grep -inE "$BREAKING_PATTERNS" "$file" | sed 's/^/  /' || true
+              BREAKING=$((BREAKING + MATCH_COUNT))
             fi
-            if echo "$CONTENT" | grep -iE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" >/dev/null; then
+
+            # Check for warnings
+            if grep -iEq "$WARNING_PATTERNS" "$file"; then
               echo "WARNING: $file"
-              echo "$CONTENT" | grep -inE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" | sed 's/^/  /'
-              WARNINGS=$((WARNINGS + 1))
+              MATCH_COUNT=$(grep -icE "$WARNING_PATTERNS" "$file")
+              grep -inE "$WARNING_PATTERNS" "$file" | sed 's/^/  /' || true
+              WARNINGS=$((WARNINGS + MATCH_COUNT))
             fi
           done <<< "$NEW_MIGRATIONS"
 
           echo "breaking_changes=$BREAKING" >> $GITHUB_OUTPUT
           echo "warnings=$WARNINGS" >> $GITHUB_OUTPUT
-        continue-on-error: true
+
+          if [[ $BREAKING -gt 0 ]]; then
+            echo "::error::Breaking changes detected in V2 database migrations."
+            echo "::error::Found $BREAKING breaking change(s)."
+            exit 1
+          fi
 
       - name: Validate V2 compatible migrations
+        if: success() || failure()
         id: v2_compat_migration_check
         run: |
           if [[ ! -d "v2_compatible_migrations" ]]; then
@@ -222,39 +275,65 @@ jobs:
           fi
 
           BASE_REF="origin/${{ github.event.pull_request.base.ref }}"
-          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "v2_compatible_migrations/**/*.sql" 2>/dev/null || echo "")
+          NEW_MIGRATIONS=$(git diff --name-only --diff-filter=AM "$BASE_REF"...HEAD -- "v2_compatible_migrations/**/up.sql" 2>/dev/null || echo "")
 
           if [[ -z "$NEW_MIGRATIONS" ]]; then
             echo "breaking_changes=0" >> $GITHUB_OUTPUT
             echo "warnings=0" >> $GITHUB_OUTPUT
             exit 0
           fi
 
+          # Load migration rules using yq
+          RULES_FILE=".github/api-migration-compatibility/migration-rules.yaml"
+
+          # Extract breaking rule patterns for v2_compatible_migrations folder
+          BREAKING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.v2_compatible_migrations.breaking[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
+          # Extract warning rule patterns for v2_compatible_migrations folder
+          WARNING_PATTERNS=$(
+            for rule in $(yq '.folder_rules.v2_compatible_migrations.warnings[]' "$RULES_FILE"); do
+              yq ".rule_patterns.$rule.pattern" "$RULES_FILE"
+            done | paste -sd'|' -
+          )
+
           BREAKING=0
           WARNINGS=0
 
           while IFS= read -r file; do
             [[ -z "$file" ]] && continue
-            CONTENT=$(cat "$file")
 
-            if echo "$CONTENT" | grep -iE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" >/dev/null; then
+            # Check for breaking changes
+            if grep -iEq "$BREAKING_PATTERNS" "$file"; then
               echo "BREAKING: $file"
-              echo "$CONTENT" | grep -inE "(DROP TABLE|DROP COLUMN|DELETE FROM|TRUNCATE)" | sed 's/^/  /'
-              BREAKING=$((BREAKING + 1))
+              MATCH_COUNT=$(grep -icE "$BREAKING_PATTERNS" "$file")
+              grep -inE "$BREAKING_PATTERNS" "$file" | sed 's/^/  /' || true
+              BREAKING=$((BREAKING + MATCH_COUNT))
             fi
 
-            if echo "$CONTENT" | grep -iE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" >/dev/null; then
+            # Check for warnings
+            if grep -iEq "$WARNING_PATTERNS" "$file"; then
               echo "WARNING: $file"
-              echo "$CONTENT" | grep -inE "(ALTER COLUMN .* TYPE|ALTER COLUMN .* SET NOT NULL|DROP INDEX|DROP CONSTRAINT)" | sed 's/^/  /'
-              WARNINGS=$((WARNINGS + 1))
+              MATCH_COUNT=$(grep -icE "$WARNING_PATTERNS" "$file")
+              grep -inE "$WARNING_PATTERNS" "$file" | sed 's/^/  /' || true
+              WARNINGS=$((WARNINGS + MATCH_COUNT))
             fi
           done <<< "$NEW_MIGRATIONS"
 
           echo "breaking_changes=$BREAKING" >> $GITHUB_OUTPUT
           echo "warnings=$WARNINGS" >> $GITHUB_OUTPUT
-        continue-on-error: true
+
+          if [[ $BREAKING -gt 0 ]]; then
+            echo "::error::Breaking changes detected in V2 compatible database migrations."
+            echo "::error::Found $BREAKING breaking change(s)."
+            exit 1
+          fi
 
       - name: Calculate migration totals
+        if: always()
         id: migration_report
         env:
           MIGRATION_BREAKING: ${{ steps.migration_check.outputs.breaking_changes }}
@@ -301,9 +380,7 @@ jobs:
           echo "Status: ${{ steps.migration_report.outputs.validation_status }}"
           echo "=========================================="
 
-      - name: Fail if migration breaking changes detected
-        if: steps.migration_report.outputs.total_breaking != '0' && steps.migration_report.outputs.total_breaking != ''
-        run: |
-          echo "::error::Breaking changes detected in database migrations."
-          echo "::error::Found ${{ steps.migration_report.outputs.total_breaking }} breaking change(s)."
-          exit 1
+          if [[ "${{ steps.migration_report.outputs.total_breaking }}" != "0" && "${{ steps.migration_report.outputs.total_breaking }}" != "" ]]; then
+            echo "::error::Migration validation failed with breaking changes."
+            exit 1
+          fi