feat: add chaos engineering & resilience testing framework

.gitattributes

@@ -0,0 +1,2 @@
+# Ensure shell scripts always use LF line endings (required by Linux containers)
+*.sh text eol=lf

.github/workflows/ci.yml

@@ -49,3 +49,73 @@ jobs:
       # - run: pnpm exec nx-cloud record -- echo Hello World
       # Nx Affected runs only tasks affected by the changes in this PR/commit. Learn more: https://nx.dev/ci/features/affected
       - run: pnpm exec nx affected -t lint test build
+
+  chaos:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        app: [url-shortener, rate-limiter, web-crawler]
+    name: chaos / ${{ matrix.app }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: tree:0
+          fetch-depth: 0
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        with:
+          run_install: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'pnpm'
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Generate Prisma client
+        if: matrix.app == 'url-shortener'
+        run: pnpm exec nx run @apps/url-shortener:prisma-generate
+
+      - name: Start chaos infrastructure
+        run: docker compose -f apps/${{ matrix.app }}/docker-compose.chaos.yml up -d --wait
+        timeout-minutes: 5
+
+      - name: Wait for Toxiproxy
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:8474/version > /dev/null 2>&1; then
+              echo "Toxiproxy is ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Toxiproxy failed to start"
+          docker compose -f apps/${{ matrix.app }}/docker-compose.chaos.yml logs
+          exit 1
+
+      - name: Create Toxiproxy proxies for migrations
+        if: matrix.app == 'url-shortener'
+        run: |
+          curl -sf -X POST http://localhost:8474/proxies \
+            -H 'Content-Type: application/json' \
+            -d '{"name":"postgres","listen":"0.0.0.0:5433","upstream":"db:5432"}'
+          curl -sf -X POST http://localhost:8474/proxies \
+            -H 'Content-Type: application/json' \
+            -d '{"name":"redis","listen":"0.0.0.0:6380","upstream":"redis:6379"}'
+
+      - name: Run Prisma migrations
+        if: matrix.app == 'url-shortener'
+        run: pnpm exec nx run @apps/url-shortener:prisma-deploy
+        env:
+          DATABASE_URL: postgresql://user:pass@localhost:5433/app
+
+      - name: Run chaos tests
+        run: pnpm exec jest --config apps/${{ matrix.app }}/jest.chaos.config.js --runInBand --forceExit
+        timeout-minutes: 5
+
+      - name: Teardown
+        if: always()
+        run: docker compose -f apps/${{ matrix.app }}/docker-compose.chaos.yml down -v

apps/rate-limiter/docker-compose.chaos.yml

@@ -0,0 +1,36 @@
+services:
+  toxiproxy:
+    image: ghcr.io/shopify/toxiproxy:latest
+    ports:
+      - '8474:8474' # Toxiproxy API
+      # Expose proxy on port 6379 — the same port the Redis Cluster advertises
+      # via REDIS_CLUSTER_ANNOUNCE_IP=127.0.0.1. This ensures the cluster client's
+      # CLUSTER SLOTS redirect to 127.0.0.1:6379 still routes through Toxiproxy.
+      - '6379:6379' # Proxy → Redis Cluster
+    healthcheck:
+      test: ['CMD', '/toxiproxy-cli', 'list']
+      interval: 3s
+      timeout: 5s
+      retries: 10
+    depends_on:
+      redis-cluster:
+        condition: service_healthy
+
+  redis-cluster:
+    image: docker.io/bitnamilegacy/redis-cluster:8.0
+    environment:
+      - 'ALLOW_EMPTY_PASSWORD=yes'
+      - 'REDIS_CLUSTER_REPLICAS=0'
+      - 'REDIS_NODES=127.0.0.1 127.0.0.1 127.0.0.1'
+      - 'REDIS_CLUSTER_CREATOR=yes'
+      - 'REDIS_CLUSTER_DYNAMIC_IPS=no'
+      - 'REDIS_CLUSTER_ANNOUNCE_IP=127.0.0.1'
+    # No host port binding — only Toxiproxy reaches Redis via Docker DNS.
+    # This prevents port conflicts and ensures all traffic goes through Toxiproxy.
+    expose:
+      - '6379'
+    healthcheck:
+      test: ['CMD', 'redis-cli', '-c', 'cluster', 'info']
+      interval: 3s
+      timeout: 5s
+      retries: 20

apps/rate-limiter/jest.chaos.config.js

@@ -0,0 +1,22 @@
+const { readFileSync } = require('fs');
+const { resolve } = require('path');
+
+const swcJestConfig = JSON.parse(
+  readFileSync(resolve(__dirname, '.spec.swcrc'), 'utf-8'),
+);
+
+swcJestConfig.swcrc = false;
+
+module.exports = {
+  displayName: '@apps/rate-limiter:chaos',
+  preset: '../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['@swc/jest', swcJestConfig],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: 'test-output/jest/coverage-chaos',
+  testMatch: ['**/*.chaos.spec.ts'],
+  testTimeout: 60_000,
+  maxWorkers: 1,
+};

apps/rate-limiter/jest.config.cts

@@ -18,5 +18,5 @@ module.exports = {
   },
   moduleFileExtensions: ['ts', 'js', 'html'],
   coverageDirectory: 'test-output/jest/coverage',
-  testMatch: ['**/*.spec.ts', '!**/*.int.spec.ts'],
+  testMatch: ['**/*.spec.ts', '!**/*.int.spec.ts', '!**/*.chaos.spec.ts'],
 };

apps/rate-limiter/package.json

@@ -87,6 +87,18 @@
           "passWithNoTests": true
         }
       },
+      "chaos": {
+        "executor": "nx:run-commands",
+        "options": {
+          "commands": [
+            "docker compose -f docker-compose.chaos.yml up -d",
+            "pnpm exec jest --config jest.chaos.config.js --runInBand --forceExit",
+            "docker compose -f docker-compose.chaos.yml down"
+          ],
+          "cwd": "apps/rate-limiter",
+          "parallel": false
+        }
+      },
       "infra:up": {
         "executor": "nx:run-commands",
         "options": {