diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 532dc31..f6918e8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,37 +22,10 @@ jobs: verify-ssm-parameters: name: Verify SSM Parameters needs: determine-environment - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ needs.determine-environment.outputs.environment_name == 'dev' && secrets.DEV_AWS_ACCESS_KEY_ID || secrets.PROD_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ needs.determine-environment.outputs.environment_name == 'dev' && secrets.DEV_AWS_SECRET_ACCESS_KEY || secrets.PROD_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Check required SSM parameters - run: | - REQUIRED_PARAMS=( - "/jaildata/alert-email" - ) - - MISSING_PARAMS=0 - - for param in "${REQUIRED_PARAMS[@]}"; do - echo "Checking SSM parameter: $param" - if ! aws ssm get-parameter --name "$param" --with-decryption 2>/dev/null; then - echo "::error::Missing required SSM parameter: $param" - MISSING_PARAMS=1 - fi - done - - if [ $MISSING_PARAMS -ne 0 ]; then - echo "::error::One or more required SSM parameters are missing" - exit 1 - fi - - echo "All required SSM parameters are present" + uses: ./.github/workflows/verify-ssm-parameters.yml + with: + environment: ${{ needs.determine-environment.outputs.environment_name }} + secrets: inherit terraform-apply: name: Terraform Apply diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 6ded403..c23b21e 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -30,37 +30,10 @@ on: jobs: verify-ssm-parameters: name: Verify SSM Parameters - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ github.event.inputs.environment == 'dev' && secrets.DEV_AWS_ACCESS_KEY_ID || secrets.PROD_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ github.event.inputs.environment == 'dev' && secrets.DEV_AWS_SECRET_ACCESS_KEY || secrets.PROD_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Check required SSM parameters - run: | - REQUIRED_PARAMS=( - "/jaildata/alert-email" - ) - - MISSING_PARAMS=0 - - for param in "${REQUIRED_PARAMS[@]}"; do - echo "Checking SSM parameter: $param" - if ! aws ssm get-parameter --name "$param" --with-decryption 2>/dev/null; then - echo "::error::Missing required SSM parameter: $param" - MISSING_PARAMS=1 - fi - done - - if [ $MISSING_PARAMS -ne 0 ]; then - echo "::error::One or more required SSM parameters are missing" - exit 1 - fi - - echo "All required SSM parameters are present" + uses: ./.github/workflows/verify-ssm-parameters.yml + with: + environment: ${{ github.event.inputs.environment }} + secrets: inherit terraform-apply: name: Terraform Apply diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 2a51e7e..9b1e917 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -33,10 +33,27 @@ jobs: - name: Check TypeScript build run: npx tsc --noEmit + verify-ssm-parameters-dev: + name: Verify SSM Parameters (Dev) + if: github.base_ref == 'main' + uses: ./.github/workflows/verify-ssm-parameters.yml + with: + environment: dev + secrets: inherit + + verify-ssm-parameters-prod: + name: Verify SSM Parameters (Prod) + if: github.base_ref == 'live' + uses: ./.github/workflows/verify-ssm-parameters.yml + with: + environment: prod + secrets: inherit + terraform-plan-dev: name: Terraform Plan (Dev) if: github.base_ref == 'main' runs-on: ubuntu-latest + needs: verify-ssm-parameters-dev defaults: run: working-directory: ./infra/terraform @@ -67,8 +84,39 @@ jobs: - name: Terraform Plan working-directory: ./infra/terraform/dev + timeout-minutes: 5 run: | - terraform plan -no-color > plan_full.txt || { exit_code=$?; cat plan_full.txt; echo "plan<> $GITHUB_OUTPUT; cat plan_full.txt >> $GITHUB_OUTPUT; echo "EOF" >> $GITHUB_OUTPUT; echo "has_changes=true" >> $GITHUB_OUTPUT; exit $exit_code; } + echo "Starting Terraform plan at $(date)" + echo "Current working directory: $(pwd)" + echo "Terraform version: $(terraform version)" + echo "AWS region: $AWS_REGION" + echo "Checking state lock status..." + + # Add debugging for state backend + terraform show -json 2>/dev/null | jq -r '.backend // "No backend info"' || echo "Could not query backend info" + + # Enable verbose logging + export TF_LOG=DEBUG + export TF_LOG_PATH=/tmp/terraform.log + + echo "Running terraform plan with timeout and debugging..." + timeout 240 terraform plan -no-color -lock-timeout=60s > plan_full.txt 2>&1 || { + exit_code=$? + echo "Terraform plan failed with exit code: $exit_code" + echo "=== Plan output ===" + cat plan_full.txt + echo "=== Terraform debug logs (last 50 lines) ===" + tail -n 50 /tmp/terraform.log 2>/dev/null || echo "No debug logs available" + echo "=== Current processes ===" + ps aux | grep -E "(terraform|aws)" | grep -v grep || echo "No terraform/aws processes found" + echo "plan<> $GITHUB_OUTPUT + cat plan_full.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "has_changes=true" >> $GITHUB_OUTPUT + exit $exit_code + } + + echo "Terraform plan completed successfully at $(date)" cat plan_full.txt # Check if there are any changes planned @@ -108,6 +156,7 @@ jobs: name: Terraform Plan (Prod) if: github.base_ref == 'live' runs-on: ubuntu-latest + needs: verify-ssm-parameters-prod environment: prod defaults: run: @@ -133,14 +182,67 @@ jobs: working-directory: ./infra/terraform/prod run: terraform init + - name: Check and Clear State Locks + working-directory: ./infra/terraform/prod + run: | + echo "Checking for existing state locks in prod account..." + # Try to acquire a lock briefly to see if one exists + if ! timeout 10 terraform plan -detailed-exitcode >/dev/null 2>&1; then + echo "Terraform operation failed, checking if it's a lock issue..." + + # Check the state file directly for lock info + if aws s3 cp s3://jaildata-tf-state/terraform.tfstate - 2>/dev/null | jq -e '.lineage' >/dev/null 2>&1; then + echo "State file exists and is readable" + else + echo "State file may have issues" + fi + + # Try to force unlock any existing locks + echo "Attempting to clear any existing locks..." + terraform force-unlock -force $(aws s3 cp s3://jaildata-tf-state/.terraform/terraform.tfstate.lock.info - 2>/dev/null | jq -r '.ID // empty') 2>/dev/null || echo "No lock file found or unable to unlock" + else + echo "No lock detected" + fi + - name: Set Terraform environment variables run: | echo "TF_VAR_alert_email=${{ vars.ALERT_EMAIL }}" >> $GITHUB_ENV - name: Terraform Plan working-directory: ./infra/terraform/prod + timeout-minutes: 5 run: | - terraform plan -no-color > plan_full.txt || { exit_code=$?; cat plan_full.txt; echo "plan<> $GITHUB_OUTPUT; cat plan_full.txt >> $GITHUB_OUTPUT; echo "EOF" >> $GITHUB_OUTPUT; echo "has_changes=true" >> $GITHUB_OUTPUT; exit $exit_code; } + echo "Starting Terraform plan at $(date)" + echo "Current working directory: $(pwd)" + echo "Terraform version: $(terraform version)" + echo "AWS region: $AWS_REGION" + echo "Checking state lock status..." + + # Add debugging for state backend + terraform show -json 2>/dev/null | jq -r '.backend // "No backend info"' || echo "Could not query backend info" + + # Enable verbose logging + export TF_LOG=DEBUG + export TF_LOG_PATH=/tmp/terraform.log + + echo "Running terraform plan with timeout and debugging..." + timeout 240 terraform plan -no-color -lock-timeout=60s > plan_full.txt 2>&1 || { + exit_code=$? + echo "Terraform plan failed with exit code: $exit_code" + echo "=== Plan output ===" + cat plan_full.txt + echo "=== Terraform debug logs (last 50 lines) ===" + tail -n 50 /tmp/terraform.log 2>/dev/null || echo "No debug logs available" + echo "=== Current processes ===" + ps aux | grep -E "(terraform|aws)" | grep -v grep || echo "No terraform/aws processes found" + echo "plan<> $GITHUB_OUTPUT + cat plan_full.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "has_changes=true" >> $GITHUB_OUTPUT + exit $exit_code + } + + echo "Terraform plan completed successfully at $(date)" cat plan_full.txt # Check if there are any changes planned diff --git a/.github/workflows/verify-ssm-parameters.yml b/.github/workflows/verify-ssm-parameters.yml new file mode 100644 index 0000000..f97cbb6 --- /dev/null +++ b/.github/workflows/verify-ssm-parameters.yml @@ -0,0 +1,80 @@ +name: Verify SSM Parameters + +on: + workflow_call: + inputs: + environment: + description: "Environment to verify (dev/prod)" + required: true + type: string + aws-region: + description: "AWS Region" + required: false + type: string + default: "us-east-2" + secrets: + DEV_AWS_ACCESS_KEY_ID: + description: "AWS Access Key ID for dev environment" + required: false + DEV_AWS_SECRET_ACCESS_KEY: + description: "AWS Secret Access Key for dev environment" + required: false + PROD_AWS_ACCESS_KEY_ID: + description: "AWS Access Key ID for prod environment" + required: false + PROD_AWS_SECRET_ACCESS_KEY: + description: "AWS Secret Access Key for prod environment" + required: false + +jobs: + verify-ssm-parameters: + name: Verify SSM Parameters (${{ inputs.environment }}) + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.environment == 'dev' && secrets.DEV_AWS_ACCESS_KEY_ID || secrets.PROD_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ inputs.environment == 'dev' && secrets.DEV_AWS_SECRET_ACCESS_KEY || secrets.PROD_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws-region }} + + - name: Check required SSM parameters + run: | + REQUIRED_PARAMS=( + "/jaildata/base-url" + "/jaildata/alert-email" + "/jaildata/alert-topic-arn" + "/jaildata/facilities/buncombe/api-id" + ) + + MISSING_PARAMS=0 + + echo "🔍 Verifying SSM parameters for ${{ inputs.environment }} environment..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + for param in "${REQUIRED_PARAMS[@]}"; do + echo "Checking SSM parameter: $param" + if ! PARAM_VALUE=$(aws ssm get-parameter --name "$param" --with-decryption --query 'Parameter.Value' --output text 2>/dev/null); then + echo "::error::Missing required SSM parameter: $param" + MISSING_PARAMS=1 + elif [ "$PARAM_VALUE" = "CHANGE_ME" ] || [ -z "$PARAM_VALUE" ]; then + echo "::error::SSM parameter $param has not been configured (value is empty or placeholder)" + MISSING_PARAMS=1 + else + echo "✅ Parameter $param is properly configured" + fi + done + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if [ $MISSING_PARAMS -ne 0 ]; then + echo "::error::❌ One or more required SSM parameters are missing or misconfigured" + echo "" + echo "💡 To fix this:" + echo " 1. Ensure Terraform has been applied to create the parameter structure" + echo " 2. Manually set the actual values in AWS Systems Manager Parameter Store" + echo " 3. Replace any 'CHANGE_ME' placeholder values with real configuration" + exit 1 + fi + + echo "🎉 All required SSM parameters are properly configured for ${{ inputs.environment }}!" diff --git a/infra/terraform/dev/dev.tf b/infra/terraform/dev/dev.tf index 2faf972..2099492 100644 --- a/infra/terraform/dev/dev.tf +++ b/infra/terraform/dev/dev.tf @@ -1,7 +1,7 @@ module "main" { - source = "../main" - environment = var.environment - region = var.region - domain = var.domain - alert_email = var.alert_email + source = "../main" + environment = var.environment + region = var.region + domain = var.domain + alert_email = var.alert_email } \ No newline at end of file diff --git a/infra/terraform/main/parameters.tf b/infra/terraform/main/parameters.tf index 3429ad3..2905554 100644 --- a/infra/terraform/main/parameters.tf +++ b/infra/terraform/main/parameters.tf @@ -1,3 +1,25 @@ +resource "aws_ssm_parameter" "jail_data_base_url" { + name = "/jaildata/base-url" + type = "String" + value = "CHANGE_ME" + description = "Base URL for external jail data API endpoints (set manually after deployment)" + + lifecycle { + ignore_changes = [value] + } +} + +resource "aws_ssm_parameter" "buncombe_api_id" { + name = "/jaildata/facilities/buncombe/api-id" + type = "String" + value = "CHANGE_ME" + description = "API ID for Buncombe County jail data system (set manually after deployment)" + + lifecycle { + ignore_changes = [value] + } +} + # API Gateway domain outputs output "ApiDomain" { description = "API Gateway custom domain name" diff --git a/serverless/README.md b/serverless/README.md index 7d5e873..53d00f7 100644 --- a/serverless/README.md +++ b/serverless/README.md @@ -4,11 +4,11 @@ This is the backend API for the Detention Data project, built with the Serverles ## Architecture -- **AWS Lambda**: Serverless functions for API endpoints and scheduled data collection -- **Amazon DynamoDB**: NoSQL database for storing detention data -- **Amazon API Gateway**: REST API with API key authentication -- **AWS EventBridge**: Scheduled triggers for data collection -- **AWS CloudWatch**: Logging and monitoring +- **AWS Lambda**: Serverless functions for API endpoints and scheduled data collection +- **Amazon DynamoDB**: NoSQL database for storing detention data +- **Amazon API Gateway**: REST API with API key authentication +- **AWS EventBridge**: Scheduled triggers for data collection +- **AWS CloudWatch**: Logging and monitoring ## Project Structure @@ -27,13 +27,13 @@ serverless/ ## API Endpoints -- `GET /status` - Health check -- `GET /detainee/{detaineeId}` - Get detainee records -- `GET /detainees/active` - List active detainees +- `GET /status` - Health check +- `GET /detainee/{detaineeId}` - Get detainee records +- `GET /detainees/active` - List active detainees ## Scheduled Functions -- **Data Collection**: Runs daily at 10 AM UTC to collect detention data from configured county sources +- **Data Collection**: Runs daily at 10 AM UTC to collect detention data from configured county sources ## Development diff --git a/serverless/api/handlers/__tests__/batch-processing.test.ts b/serverless/api/handlers/__tests__/batch-processing.test.ts new file mode 100644 index 0000000..d51883d --- /dev/null +++ b/serverless/api/handlers/__tests__/batch-processing.test.ts @@ -0,0 +1,222 @@ +import { SQSEvent, SQSRecord } from "aws-lambda"; +import { processBatch } from "../batch-processing"; +import StorageClient from "../../../lib/StorageClient"; + +// Mock StorageClient +jest.mock("../../../lib/StorageClient", () => ({ + batchSaveInmates: jest.fn(), +})); + +// Mock AlertService +jest.mock("../../../lib/AlertService", () => ({ + forCategory: jest.fn().mockReturnValue({ + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }), + AlertCategory: { + BATCH_PROCESSING: "BATCH_PROCESSING", + }, +})); + +const mockStorageClient = StorageClient as jest.Mocked; + +describe("Batch Processing Handler", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const createMockSQSRecord = ( + facilityId: string, + batchNumber: number, + inmates: any[] + ): SQSRecord => ({ + messageId: `message-${batchNumber}`, + receiptHandle: "receipt-handle", + body: JSON.stringify({ + facilityId, + batch: { + Inmates: inmates, + Total: inmates.length, + ShowImages: false, + }, + batchNumber, + totalBatches: 1, + requestId: "test-request-id", + }), + attributes: { + ApproximateReceiveCount: "1", + SentTimestamp: "1234567890", + SenderId: "sender-id", + ApproximateFirstReceiveTimestamp: "1234567890", + }, + messageAttributes: {}, + md5OfBody: "md5-hash", + eventSource: "aws:sqs", + eventSourceARN: "arn:aws:sqs:us-east-1:123456789012:test-queue", + awsRegion: "us-east-1", + }); + + it("should process a single batch successfully", async () => { + const inmates = [ + { + FirstName: "John", + LastName: "Doe", + ArrestDate: "9/15/2025 10:30:00 AM", + }, + { + FirstName: "Jane", + LastName: "Smith", + ArrestDate: "9/16/2025 11:00:00 AM", + }, + ]; + + const sqsEvent: SQSEvent = { + Records: [createMockSQSRecord("wake", 1, inmates)], + }; + + mockStorageClient.batchSaveInmates.mockResolvedValueOnce(undefined); + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledTimes(1); + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledWith( + "wake", + inmates + ); + }); + + it("should process multiple batches", async () => { + const inmates1 = [ + { + FirstName: "John", + LastName: "Doe", + ArrestDate: "9/15/2025 10:30:00 AM", + }, + ]; + const inmates2 = [ + { + FirstName: "Jane", + LastName: "Smith", + ArrestDate: "9/16/2025 11:00:00 AM", + }, + ]; + + const sqsEvent: SQSEvent = { + Records: [ + createMockSQSRecord("wake", 1, inmates1), + createMockSQSRecord("buncombe", 2, inmates2), + ], + }; + + mockStorageClient.batchSaveInmates.mockResolvedValue(undefined); + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledTimes(2); + expect(mockStorageClient.batchSaveInmates).toHaveBeenNthCalledWith( + 1, + "wake", + inmates1 + ); + expect(mockStorageClient.batchSaveInmates).toHaveBeenNthCalledWith( + 2, + "buncombe", + inmates2 + ); + }); + + it("should handle empty batch gracefully", async () => { + const sqsEvent: SQSEvent = { + Records: [createMockSQSRecord("wake", 1, [])], + }; + + mockStorageClient.batchSaveInmates.mockResolvedValueOnce(undefined); + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledTimes(1); + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledWith( + "wake", + [] + ); + }); + + it("should continue processing other batches when one fails", async () => { + const inmates1 = [{ FirstName: "John", LastName: "Doe" }]; + const inmates2 = [{ FirstName: "Jane", LastName: "Smith" }]; + + const sqsEvent: SQSEvent = { + Records: [ + createMockSQSRecord("wake", 1, inmates1), + createMockSQSRecord("buncombe", 2, inmates2), + ], + }; + + mockStorageClient.batchSaveInmates + .mockRejectedValueOnce(new Error("Storage error")) + .mockResolvedValueOnce(undefined); + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledTimes(2); + }); + + it("should handle malformed SQS message body", async () => { + const malformedRecord: SQSRecord = { + ...createMockSQSRecord("wake", 1, []), + body: "invalid json", + }; + + const sqsEvent: SQSEvent = { + Records: [malformedRecord], + }; + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).not.toHaveBeenCalled(); + }); + + it("should handle missing facility ID in message", async () => { + const recordWithoutFacility: SQSRecord = { + ...createMockSQSRecord("wake", 1, []), + body: JSON.stringify({ + batch: { Inmates: [], Total: 0, ShowImages: false }, + batchNumber: 1, + requestId: "test-request-id", + // Missing facilityId + }), + }; + + const sqsEvent: SQSEvent = { + Records: [recordWithoutFacility], + }; + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).not.toHaveBeenCalled(); + }); + + it("should handle large batch with many inmates", async () => { + // Create a batch with 100 inmates + const inmates = Array.from({ length: 100 }, (_, i) => ({ + FirstName: `Inmate${i}`, + LastName: "Test", + ArrestDate: "9/15/2025 10:30:00 AM", + })); + + const sqsEvent: SQSEvent = { + Records: [createMockSQSRecord("wake", 1, inmates)], + }; + + mockStorageClient.batchSaveInmates.mockResolvedValueOnce(undefined); + + await processBatch(sqsEvent); + + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledTimes(1); + expect(mockStorageClient.batchSaveInmates).toHaveBeenCalledWith( + "wake", + inmates + ); + }); +}); diff --git a/serverless/api/handlers/__tests__/data-collection.test.ts b/serverless/api/handlers/__tests__/data-collection.test.ts new file mode 100644 index 0000000..d2b9a3c --- /dev/null +++ b/serverless/api/handlers/__tests__/data-collection.test.ts @@ -0,0 +1,374 @@ +import { APIGatewayProxyEvent, ScheduledEvent } from "aws-lambda"; + +// Mock external dependencies using ZipCase pattern BEFORE importing the module +jest.mock("axios"); + +// Mock AWS SDK clients +const mockSQSSend = jest.fn(); +const mockSSMSend = jest.fn(); + +jest.mock("@aws-sdk/client-sqs", () => ({ + SQSClient: jest.fn().mockImplementation(() => ({ + send: mockSQSSend, + })), + SendMessageCommand: jest.fn(), +})); + +jest.mock("@aws-sdk/client-ssm", () => ({ + SSMClient: jest.fn().mockImplementation(() => ({ + send: mockSSMSend, + })), + GetParameterCommand: jest.fn(), +})); + +jest.mock("uuid", () => ({ + v4: jest.fn(() => "mock-uuid-123"), +})); + +// Mock AlertService +jest.mock("../../../lib/AlertService", () => ({ + __esModule: true, + default: { + forCategory: jest.fn().mockReturnValue({ + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }), + }, + AlertCategory: { + DATA_COLLECTION: "DATA_COLLECTION", + }, +})); + +// Mock FacilityMapping +jest.mock("../../../lib/FacilityMapping", () => ({ + FacilityMapper: { + loadApiIds: jest.fn(), + getFacilityByName: jest.fn(), + }, +})); + +// Import axios after mocking +import axios from "axios"; +const mockedAxios = axios as jest.Mocked; + +import { collect, collectScheduled } from "../data-collection"; +import { FacilityMapper } from "../../../lib/FacilityMapping"; +const mockFacilityMapper = FacilityMapper as jest.Mocked; + +// Helper functions for test events +const createMockEvent = ( + facilityId?: string, + body?: string +): APIGatewayProxyEvent => ({ + pathParameters: facilityId ? { facilityId } : null, + body: body || null, + headers: {}, + multiValueHeaders: {}, + httpMethod: "POST", + isBase64Encoded: false, + path: `/collect/${facilityId}`, + resource: "/collect/{facilityId}", + requestContext: {} as any, + stageVariables: null, + queryStringParameters: null, + multiValueQueryStringParameters: null, +}); + +const createMockScheduledEvent = (facilityId: string): ScheduledEvent => ({ + version: "0", + id: "scheduled-event-id", + "detail-type": "Scheduled Event", + source: "aws.events", + account: "123456789012", + time: "2025-09-29T10:00:00Z", + region: "us-east-1", + detail: { facilityId }, + resources: ["arn:aws:events:us-east-1:123456789012:rule/test-rule"], +}); + +describe("Data Collection Handler", () => { + beforeEach(() => { + jest.clearAllMocks(); + + // Reset environment variables + process.env.AWS_REGION = "us-east-1"; + process.env.AWS_ACCOUNT_ID = "123456789012"; + process.env.STAGE = "test"; + + // Set up SSM client mock to return successful responses by default + mockSSMSend.mockResolvedValue({ + Parameter: { + Value: "https://api.example.com", + }, + }); + + // Set up axios mock + mockedAxios.create.mockReturnValue({ + get: jest.fn(), + post: jest.fn(), + interceptors: { + request: { use: jest.fn() }, + response: { use: jest.fn() }, + }, + } as any); + + // Set up default successful facility mapping + mockFacilityMapper.getFacilityByName.mockReturnValue({ + name: "wake", + apiId: 384, + displayName: "Wake County", + }); + }); + + describe("Manual Collection (collect)", () => { + it("should start data collection successfully", async () => { + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(202); + const body = JSON.parse(result.body); + expect(body.message).toBe("Data collection started"); + expect(body.facilityId).toBe("wake"); + expect(mockSSMSend).toHaveBeenCalledTimes(1); + }); + + it("should return 400 for missing facilityId", async () => { + const event = createMockEvent(); + const result = await collect(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error).toBe("facilityId is required"); + expect(mockSSMSend).not.toHaveBeenCalled(); + }); + + it("should handle facilityId from request body", async () => { + const event = createMockEvent( + undefined, + JSON.stringify({ facilityId: "buncombe" }) + ); + const result = await collect(event); + + expect(result.statusCode).toBe(202); + const body = JSON.parse(result.body); + expect(body.facilityId).toBe("buncombe"); + }); + + it("should handle unknown facility", async () => { + mockFacilityMapper.getFacilityByName.mockReturnValueOnce(null); + + const event = createMockEvent("unknown"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + expect(body.message).toBe("Unknown facility: unknown"); + }); + + it("should handle facility without API ID", async () => { + mockFacilityMapper.getFacilityByName.mockReturnValueOnce({ + name: "mecklenburg", + apiId: 0, + displayName: "Mecklenburg County", + }); + + const event = createMockEvent("mecklenburg"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + expect(body.message).toBe( + "Facility mecklenburg does not have a configured API ID" + ); + }); + + it("should handle SSM parameter not found", async () => { + mockSSMSend.mockResolvedValueOnce({ + Parameter: null, + }); + + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + + it("should handle SSM parameter missing value", async () => { + mockSSMSend.mockResolvedValueOnce({ + Parameter: { + Value: null, + }, + }); + + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + + it("should handle SSM client errors", async () => { + mockSSMSend.mockRejectedValueOnce(new Error("SSM Error")); + + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + + it("should handle malformed JSON in request body", async () => { + const event = createMockEvent(undefined, "invalid json"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + }); + + describe("Scheduled Collection (collectScheduled)", () => { + it("should process scheduled collection and handle initialization", async () => { + const event = createMockScheduledEvent("wake"); + + // The scheduled function should call the DataCollectionService which will attempt + // to initialize the session. Since we can't easily mock the full session flow, + // we expect it to fail with the XSRF token error which means it got past + // the facility validation and SSM parameter retrieval + await expect(collectScheduled(event)).rejects.toThrow( + "Failed to obtain XSRF token from initial request" + ); + + expect(mockSSMSend).toHaveBeenCalledTimes(1); + }); + + it("should handle missing facilityId in event detail", async () => { + const event: ScheduledEvent = { + ...createMockScheduledEvent("wake"), + detail: {}, + }; + + await expect(collectScheduled(event)).rejects.toThrow( + "facilityId is required in event input" + ); + }); + + it("should handle unknown facility in scheduled event", async () => { + mockFacilityMapper.getFacilityByName.mockReturnValueOnce(null); + + const event = createMockScheduledEvent("unknown"); + + await expect(collectScheduled(event)).rejects.toThrow( + "Unknown facility: unknown" + ); + }); + + it("should handle facility without API ID in scheduled event", async () => { + mockFacilityMapper.getFacilityByName.mockReturnValueOnce({ + name: "mecklenburg", + apiId: 0, + displayName: "Mecklenburg County", + }); + + const event = createMockScheduledEvent("mecklenburg"); + + await expect(collectScheduled(event)).rejects.toThrow( + "Facility mecklenburg does not have a configured API ID" + ); + }); + + it("should handle SSM errors in scheduled collection", async () => { + mockSSMSend.mockRejectedValueOnce(new Error("SSM Error")); + + const event = createMockScheduledEvent("wake"); + + await expect(collectScheduled(event)).rejects.toThrow("SSM Error"); + }); + + it("should handle SSM parameter not found", async () => { + mockSSMSend.mockResolvedValueOnce({ + Parameter: null, + }); + + const event = createMockScheduledEvent("wake"); + + await expect(collectScheduled(event)).rejects.toThrow( + "Base URL not configured in SSM Parameter Store" + ); + }); + }); + + describe("Configuration and Validation", () => { + it("should validate facilityId parameter before SSM call", async () => { + const event = createMockEvent("wake"); + await collect(event); + + expect(mockFacilityMapper.getFacilityByName).toHaveBeenCalledWith( + "wake" + ); + expect(mockSSMSend).toHaveBeenCalledTimes(1); + }); + + it("should create axios instance with proper configuration", async () => { + const event = createMockEvent("wake"); + await collect(event); + + expect(mockedAxios.create).toHaveBeenCalledWith( + expect.objectContaining({ + baseURL: "https://api.example.com", + timeout: 30000, + headers: expect.objectContaining({ + "User-Agent": "Mozilla/5.0 (compatible; JailData/1.0)", + Accept: "application/json, text/plain, */*", + "Accept-Language": "en-US,en;q=0.9", + }), + }) + ); + }); + + it("should set up axios interceptors for cookie handling", async () => { + const event = createMockEvent("wake"); + await collect(event); + + const axiosInstance = mockedAxios.create.mock.results[0].value; + expect(axiosInstance.interceptors.request.use).toHaveBeenCalled(); + expect(axiosInstance.interceptors.response.use).toHaveBeenCalled(); + }); + }); + + describe("Error Handling", () => { + it("should handle unexpected errors gracefully", async () => { + // Force an unexpected error by making FacilityMapper throw + mockFacilityMapper.getFacilityByName.mockImplementationOnce(() => { + throw new Error("Unexpected error"); + }); + + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + + it("should handle facility mapping service errors", async () => { + mockFacilityMapper.getFacilityByName.mockImplementationOnce(() => { + throw new Error("Service unavailable"); + }); + + const event = createMockEvent("wake"); + const result = await collect(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Failed to start data collection"); + }); + }); +}); diff --git a/serverless/api/handlers/__tests__/detainee.test.ts b/serverless/api/handlers/__tests__/detainee.test.ts new file mode 100644 index 0000000..09d0592 --- /dev/null +++ b/serverless/api/handlers/__tests__/detainee.test.ts @@ -0,0 +1,384 @@ +import { APIGatewayProxyEvent } from "aws-lambda"; +import { + getInmatesByFacility, + getAllActiveInmates, + searchInmatesByName, + getSpecificInmate, +} from "../detainee"; +import StorageClient from "../../../lib/StorageClient"; +import { FacilityMapper } from "../../../lib/FacilityMapping"; +import { InmateDynamoRecord } from "../../../lib/types"; + +// Mock StorageClient +jest.mock("../../../lib/StorageClient", () => ({ + getInmatesByFacility: jest.fn(), + getAllRecentInmates: jest.fn(), + searchInmatesByLastName: jest.fn(), + getInmate: jest.fn(), +})); + +// Mock FacilityMapping +jest.mock("../../../lib/FacilityMapping", () => ({ + FacilityMapper: { + isValidFacilityName: jest.fn(), + getAllFacilities: jest.fn(), + }, +})); + +// Mock AlertService +jest.mock("../../../lib/AlertService", () => ({ + forCategory: jest.fn().mockReturnValue({ + error: jest.fn(), + }), + AlertCategory: { + DATABASE: "DATABASE", + }, +})); + +const mockStorageClient = StorageClient as jest.Mocked; +const mockFacilityMapper = FacilityMapper as jest.Mocked; + +describe("Detainee Handler", () => { + beforeEach(() => { + jest.clearAllMocks(); + mockFacilityMapper.isValidFacilityName.mockReturnValue(true); + mockFacilityMapper.getAllFacilities.mockReturnValue([ + { name: "wake", apiId: 384, displayName: "Wake County" }, + { name: "buncombe", apiId: 23, displayName: "Buncombe County" }, + ]); + }); + + const createMockInmate = ( + overrides: Partial = {} + ): InmateDynamoRecord => ({ + PK: "INMATE#wake#DOE#JOHN#", + SK: "2025-09-15", + GSI1PK: "FACILITY#wake", + GSI1SK: "2025-09-15", + recordDate: "2025-09-15", + lastUpdated: "2025-09-15T10:30:00Z", + totalBondAmount: 1000, + rawData: { FirstName: "John", LastName: "Doe" }, + ...overrides, + }); + + describe("getInmatesByFacility", () => { + const createMockEvent = ( + facilityId?: string, + limit?: string + ): APIGatewayProxyEvent => ({ + pathParameters: facilityId ? { facilityId } : null, + queryStringParameters: limit ? { limit } : null, + body: null, + headers: {}, + multiValueHeaders: {}, + httpMethod: "GET", + isBase64Encoded: false, + path: `/inmates/${facilityId}`, + resource: "/inmates/{facilityId}", + requestContext: {} as any, + stageVariables: null, + multiValueQueryStringParameters: null, + }); + + it("should return inmates for valid facility", async () => { + const mockInmates = [createMockInmate()]; + + mockStorageClient.getInmatesByFacility.mockResolvedValueOnce( + mockInmates + ); + + const event = createMockEvent("wake"); + const result = await getInmatesByFacility(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.facilityId).toBe("wake"); + expect(body.inmates).toEqual(mockInmates); + expect(body.count).toBe(1); + expect(body.description).toBe("Recent inmates (last 24 hours)"); + }); + + it("should return 400 for missing facilityId", async () => { + const event = createMockEvent(); + const result = await getInmatesByFacility(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error).toBe("Missing facilityId parameter"); + }); + + it("should return 400 for invalid facility", async () => { + mockFacilityMapper.isValidFacilityName.mockReturnValueOnce(false); + + const event = createMockEvent("invalid"); + const result = await getInmatesByFacility(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error).toContain("Invalid facilityId: invalid"); + }); + + it("should use custom limit parameter", async () => { + mockStorageClient.getInmatesByFacility.mockResolvedValueOnce([]); + + const event = createMockEvent("wake", "50"); + await getInmatesByFacility(event); + + expect(mockStorageClient.getInmatesByFacility).toHaveBeenCalledWith( + "wake", + 50 + ); + }); + + it("should use default limit when not specified", async () => { + mockStorageClient.getInmatesByFacility.mockResolvedValueOnce([]); + + const event = createMockEvent("wake"); + await getInmatesByFacility(event); + + expect(mockStorageClient.getInmatesByFacility).toHaveBeenCalledWith( + "wake", + 100 + ); + }); + + it("should handle StorageClient errors", async () => { + mockStorageClient.getInmatesByFacility.mockRejectedValueOnce( + new Error("Storage error") + ); + + const event = createMockEvent("wake"); + const result = await getInmatesByFacility(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.error).toBe("Internal server error"); + }); + }); + + describe("getAllActiveInmates", () => { + const createMockEvent = (limit?: string): APIGatewayProxyEvent => ({ + queryStringParameters: limit ? { limit } : null, + pathParameters: null, + body: null, + headers: {}, + multiValueHeaders: {}, + httpMethod: "GET", + isBase64Encoded: false, + path: "/inmates/active", + resource: "/inmates/active", + requestContext: {} as any, + stageVariables: null, + multiValueQueryStringParameters: null, + }); + + it("should return recent inmates across all facilities", async () => { + const mockInmates = [ + createMockInmate(), + createMockInmate({ + recordDate: "2025-09-14", + SK: "2025-09-14", + }), + ]; + + mockStorageClient.getAllRecentInmates.mockResolvedValueOnce( + mockInmates + ); + + const event = createMockEvent(); + const result = await getAllActiveInmates(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.inmates).toEqual(mockInmates); + expect(body.count).toBe(2); + expect(body.description).toBe("Recent inmates (last 24 hours)"); + }); + + it("should use custom limit", async () => { + mockStorageClient.getAllRecentInmates.mockResolvedValueOnce([]); + + const event = createMockEvent("200"); + await getAllActiveInmates(event); + + expect(mockStorageClient.getAllRecentInmates).toHaveBeenCalledWith( + 200 + ); + }); + }); + + describe("searchInmatesByName", () => { + const createMockEvent = ( + facilityId?: string, + lastName?: string, + limit?: string + ): APIGatewayProxyEvent => ({ + pathParameters: facilityId ? { facilityId } : null, + queryStringParameters: { + ...(lastName && { lastName }), + ...(limit && { limit }), + }, + body: null, + headers: {}, + multiValueHeaders: {}, + httpMethod: "GET", + isBase64Encoded: false, + path: `/inmates/${facilityId}/search`, + resource: "/inmates/{facilityId}/search", + requestContext: {} as any, + stageVariables: null, + multiValueQueryStringParameters: null, + }); + + it("should search inmates by last name", async () => { + const mockInmates = [ + createMockInmate({ + rawData: { FirstName: "John", LastName: "Smith" }, + }), + ]; + + mockStorageClient.searchInmatesByLastName.mockResolvedValueOnce( + mockInmates + ); + + const event = createMockEvent("wake", "Smith"); + const result = await searchInmatesByName(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.facilityId).toBe("wake"); + expect(body.searchTerm).toBe("Smith"); + expect(body.inmates).toEqual(mockInmates); + }); + + it("should return 400 for missing lastName", async () => { + const event = createMockEvent("wake"); + const result = await searchInmatesByName(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error).toBe("Missing lastName query parameter"); + }); + + it("should return 400 for invalid facility", async () => { + mockFacilityMapper.isValidFacilityName.mockReturnValueOnce(false); + + const event = createMockEvent("invalid", "Smith"); + const result = await searchInmatesByName(event); + + expect(result.statusCode).toBe(400); + }); + }); + + describe("getSpecificInmate", () => { + const createMockEvent = (params: { + facilityId?: string; + lastName?: string; + firstName?: string; + middleName?: string; + recordDate?: string; + }): APIGatewayProxyEvent => ({ + pathParameters: { + facilityId: params.facilityId || undefined, + lastName: params.lastName || undefined, + firstName: params.firstName || undefined, + middleName: params.middleName || undefined, + recordDate: params.recordDate || undefined, + }, + queryStringParameters: null, + body: null, + headers: {}, + multiValueHeaders: {}, + httpMethod: "GET", + isBase64Encoded: false, + path: "/inmates/wake/DOE/JOHN/2025-09-15", + resource: + "/inmates/{facilityId}/{lastName}/{firstName}/{recordDate}", + requestContext: {} as any, + stageVariables: null, + multiValueQueryStringParameters: null, + }); + + it("should return specific inmate", async () => { + const mockInmate = createMockInmate(); + + mockStorageClient.getInmate.mockResolvedValueOnce(mockInmate); + + const event = createMockEvent({ + facilityId: "wake", + lastName: "Doe", + firstName: "John", + recordDate: "2025-09-15", + }); + + const result = await getSpecificInmate(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body).toEqual(mockInmate); + }); + + it("should return 404 when inmate not found", async () => { + mockStorageClient.getInmate.mockResolvedValueOnce(null); + + const event = createMockEvent({ + facilityId: "wake", + lastName: "Doe", + firstName: "John", + recordDate: "2025-09-15", + }); + + const result = await getSpecificInmate(event); + + expect(result.statusCode).toBe(404); + const body = JSON.parse(result.body); + expect(body.error).toBe("Inmate not found"); + }); + + it("should return 400 for missing required parameters", async () => { + const event = createMockEvent({ + facilityId: "wake", + // Missing lastName, firstName, recordDate + }); + + const result = await getSpecificInmate(event); + + expect(result.statusCode).toBe(400); + const body = JSON.parse(result.body); + expect(body.error).toBe( + "Missing required parameters: facilityId, lastName, firstName, recordDate" + ); + }); + + it("should handle middle name parameter", async () => { + const mockInmate = createMockInmate({ + rawData: { + FirstName: "John", + LastName: "Doe", + MiddleName: "M", + }, + }); + + mockStorageClient.getInmate.mockResolvedValueOnce(mockInmate); + + const event = createMockEvent({ + facilityId: "wake", + lastName: "Doe", + firstName: "John", + middleName: "M", + recordDate: "2025-09-15", + }); + + await getSpecificInmate(event); + + expect(mockStorageClient.getInmate).toHaveBeenCalledWith( + "wake", + "Doe", + "John", + "M", + "2025-09-15" + ); + }); + }); +}); diff --git a/serverless/api/handlers/__tests__/status.test.ts b/serverless/api/handlers/__tests__/status.test.ts new file mode 100644 index 0000000..7e9720c --- /dev/null +++ b/serverless/api/handlers/__tests__/status.test.ts @@ -0,0 +1,48 @@ +import { APIGatewayProxyEvent } from "aws-lambda"; +import { get } from "../status"; + +// Mock the environment variables +process.env.STAGE = "test"; + +describe("Status Handler", () => { + it("should return status information", async () => { + const mockEvent = {} as APIGatewayProxyEvent; + + const result = await get(mockEvent); + + expect(result.statusCode).toBe(200); + + const body = JSON.parse(result.body); + expect(body).toHaveProperty("status", "healthy"); + expect(body).toHaveProperty("service", "detention-data-api"); + expect(body).toHaveProperty("environment", "test"); + expect(body).toHaveProperty("timestamp"); + expect(body).toHaveProperty("version", "1.0.0"); + }); + + it("should return timestamp as valid ISO string", async () => { + const mockEvent = {} as APIGatewayProxyEvent; + + const result = await get(mockEvent); + const body = JSON.parse(result.body); + + expect(() => new Date(body.timestamp)).not.toThrow(); + expect(new Date(body.timestamp)).toBeInstanceOf(Date); + }); + + it("should handle missing environment variables", async () => { + const originalStage = process.env.STAGE; + delete process.env.STAGE; + + const mockEvent = {} as APIGatewayProxyEvent; + const result = await get(mockEvent); + const body = JSON.parse(result.body); + + expect(body.environment).toBe("unknown"); + + // Restore environment variable + if (originalStage) { + process.env.STAGE = originalStage; + } + }); +}); diff --git a/serverless/api/handlers/batch-processing.ts b/serverless/api/handlers/batch-processing.ts new file mode 100644 index 0000000..25990bc --- /dev/null +++ b/serverless/api/handlers/batch-processing.ts @@ -0,0 +1,86 @@ +import { SQSEvent, SQSRecord } from "aws-lambda"; +import AlertService, { AlertCategory } from "../../lib/AlertService"; +import StorageClient from "../../lib/StorageClient"; +import { BatchProcessingMessage } from "../../lib/types"; + +const alertService = AlertService.forCategory(AlertCategory.BATCH_PROCESSING); + +export const processBatch = async (event: SQSEvent): Promise => { + try { + await alertService.info( + `Processing ${event.Records.length} batch messages` + ); + + // Process each SQS record (batch message) + const results = await Promise.allSettled( + event.Records.map((record) => processSingleBatch(record)) + ); + + // Count successes and failures + const successes = results.filter( + (r) => r.status === "fulfilled" + ).length; + const failures = results.filter((r) => r.status === "rejected").length; + + if (failures > 0) { + await alertService.warn( + `Batch processing completed with ${failures} failures out of ${results.length} batches` + ); + + // Log specific failure details + results.forEach((result, index) => { + if (result.status === "rejected") { + alertService.error( + `Batch ${index} processing failed`, + result.reason as Error + ); + } + }); + } else { + await alertService.info( + `Successfully processed ${successes} batches` + ); + } + } catch (error) { + await alertService.error( + `Batch processing handler failed: ${error}`, + error as Error + ); + throw error; + } +}; + +async function processSingleBatch(record: SQSRecord): Promise { + try { + const batchMessage: BatchProcessingMessage = JSON.parse(record.body); + + // Validate required fields + if (!batchMessage.facilityId) { + throw new Error("Missing facilityId in batch message"); + } + + if (!batchMessage.batch || !Array.isArray(batchMessage.batch.Inmates)) { + throw new Error("Invalid batch structure in message"); + } + + await alertService.info( + `Processing batch ${batchMessage.batchNumber} for facility ${batchMessage.facilityId} with ${batchMessage.batch.Inmates.length} inmates` + ); + + // Process all inmates in the batch using StorageClient batch operation + await StorageClient.batchSaveInmates( + batchMessage.facilityId, + batchMessage.batch.Inmates + ); + + await alertService.info( + `Successfully processed ${batchMessage.batch.Inmates.length} inmates from batch ${batchMessage.batchNumber} for facility ${batchMessage.facilityId}` + ); + } catch (error) { + await alertService.error( + `Failed to process batch: ${error}`, + error as Error + ); + throw error; + } +} diff --git a/serverless/api/handlers/data-collection.ts b/serverless/api/handlers/data-collection.ts index 51dede4..cbe9389 100644 --- a/serverless/api/handlers/data-collection.ts +++ b/serverless/api/handlers/data-collection.ts @@ -1,141 +1,409 @@ import { - APIGatewayProxyEvent, - APIGatewayProxyResult, - ScheduledEvent, + APIGatewayProxyEvent, + APIGatewayProxyResult, + ScheduledEvent, } from "aws-lambda"; -import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; -import { DynamoDBDocumentClient, PutCommand } from "@aws-sdk/lib-dynamodb"; +import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs"; +import { SSMClient, GetParameterCommand } from "@aws-sdk/client-ssm"; +import axios, { AxiosInstance } from "axios"; +import { v4 as uuidv4 } from "uuid"; +import AlertService, { AlertCategory } from "../../lib/AlertService"; +import { FacilityMapper } from "../../lib/FacilityMapping"; +import { + DataCollectionConfig, + InmateApiRequestBody, + InmateApiResponse, + BatchProcessingMessage, +} from "../../lib/types"; -const dynamoClient = new DynamoDBClient({ region: process.env.AWS_REGION }); -const docClient = DynamoDBDocumentClient.from(dynamoClient); +const sqsClient = new SQSClient({ region: process.env.AWS_REGION }); +const ssmClient = new SSMClient({ region: process.env.AWS_REGION }); +const alertService = AlertService.forCategory(AlertCategory.DATA_COLLECTION); -interface JailDataEvent { - countyId: string; - source: string; -} +// Cookie jar to store session cookies including XSRF token +class CookieJar { + private cookies: Map = new Map(); -interface DetentionRecord { - detaineeId: string; - timestamp: string; - status: "ACTIVE" | "INACTIVE"; - createdDate: string; - countyId: string; - source: string; - // Add other fields as needed - firstName?: string; - lastName?: string; - bookingDate?: string; - charges?: string[]; - ttl?: number; -} + public setCookie(cookieString: string): void { + const cookies = cookieString.split(";"); + for (const cookie of cookies) { + const [name, value] = cookie.trim().split("="); + if (name && value) { + this.cookies.set(name, value); + } + } + } -export const execute = async ( - event: ScheduledEvent | APIGatewayProxyEvent -): Promise => { - try { - console.log("Data collection started", JSON.stringify(event, null, 2)); - - // Parse input from scheduled event or API Gateway - let inputData: JailDataEvent; - - if ("source" in event && event.source === "aws.events") { - // Scheduled event - const scheduledEvent = event as ScheduledEvent; - inputData = JSON.parse( - scheduledEvent.detail ? JSON.stringify(scheduledEvent.detail) : "{}" - ); - } else { - // API Gateway event (for manual testing) - const apiEvent = event as APIGatewayProxyEvent; - inputData = JSON.parse(apiEvent.body || "{}"); + public getCookie(name: string): string | undefined { + return this.cookies.get(name); } - const { countyId, source } = inputData; + public getCookieHeader(): string { + return Array.from(this.cookies.entries()) + .map(([name, value]) => `${name}=${value}`) + .join("; "); + } - if (!countyId || !source) { - const error = "Missing required parameters: countyId and source"; - console.error(error); + public getXsrfToken(): string | undefined { + return this.getCookie("XSRF-TOKEN"); + } +} - if ("httpMethod" in event) { - return { - statusCode: 400, - body: JSON.stringify({ error }), +export class DataCollectionService { + private axiosInstance: AxiosInstance; + private cookieJar: CookieJar; + private config: DataCollectionConfig; + private facilityApiId: number; + private facilityName: string; + + constructor( + facilityName: string, + baseUrl: string, + batchSize: number = 100 + ) { + this.facilityName = facilityName; + this.config = { + facilityId: facilityName, // Keep human-friendly name for internal use + baseUrl: baseUrl, + batchSize: batchSize, }; - } - return; + this.cookieJar = new CookieJar(); + // Note: facilityApiId will be set during initialization + this.facilityApiId = 0; + + // Create axios instance with interceptors for cookie handling + this.axiosInstance = axios.create({ + baseURL: this.config.baseUrl, + timeout: 30000, + headers: { + "User-Agent": "Mozilla/5.0 (compatible; JailData/1.0)", + Accept: "application/json, text/plain, */*", + "Accept-Language": "en-US,en;q=0.9", + }, + }); + + this.setupInterceptors(); + } + + // Initialize facility configuration (must be called after FacilityMapper.loadApiIds()) + public initializeFacility(): void { + // Get facility configuration from mapping + const facilityConfig = FacilityMapper.getFacilityByName( + this.facilityName + ); + if (!facilityConfig) { + throw new Error(`Unknown facility: ${this.facilityName}`); + } + + if (facilityConfig.apiId === 0) { + throw new Error( + `Facility ${this.facilityName} does not have a configured API ID` + ); + } + + this.facilityApiId = facilityConfig.apiId; } - // TODO: Implement actual data collection logic - // This is a stub that would be replaced with real county data scraping - const mockData = await collectJailData(countyId, source); - - // Store the collected data - const results = await Promise.all( - mockData.map((record) => storeDetentionRecord(record)) - ); - - console.log( - `Successfully processed ${results.length} records for ${countyId}` - ); - - if ("httpMethod" in event) { - return { - statusCode: 200, - body: JSON.stringify({ - message: `Successfully processed ${results.length} records`, - countyId, - source, - }), - }; + private setupInterceptors(): void { + // Intercept responses to capture cookies + this.axiosInstance.interceptors.response.use( + (response) => { + const setCookieHeader = response.headers["set-cookie"]; + if (setCookieHeader) { + setCookieHeader.forEach((cookie: string) => + this.cookieJar.setCookie(cookie)); + } + return response; + }, + (error) => { + const setCookieHeader = error.response?.headers["set-cookie"]; + if (setCookieHeader) { + setCookieHeader.forEach((cookie: string) => + this.cookieJar.setCookie(cookie)); + } + throw error; + } + ); + + // Intercept requests to add cookies and XSRF token + this.axiosInstance.interceptors.request.use((config) => { + // Add cookies to request + const cookieHeader = this.cookieJar.getCookieHeader(); + if (cookieHeader) { + config.headers["Cookie"] = cookieHeader; + } + + // Add XSRF token for POST requests + if (config.method?.toUpperCase() === "POST") { + const xsrfToken = this.cookieJar.getXsrfToken(); + if (xsrfToken) { + config.headers["X-Xsrf-Token"] = xsrfToken; + } + config.headers["Origin"] = this.config.baseUrl; + config.headers[ + "Referer" + ] = `${this.config.baseUrl}/Inmates/Catalog`; + } + + return config; + }); } - } catch (error) { - console.error("Error in data collection:", error); - - if ("httpMethod" in event) { - return { - statusCode: 500, - body: JSON.stringify({ error: "Internal server error" }), - }; + + // Initialize session by making GET request to capture cookies + public async initializeSession(): Promise { + try { + await this.axiosInstance.get(`/api/inmates/${this.facilityApiId}`); + + const xsrfToken = this.cookieJar.getXsrfToken(); + if (!xsrfToken) { + throw new Error( + "Failed to obtain XSRF token from initial request" + ); + } + + await alertService.info( + `Session initialized for facility ${this.config.facilityId}, XSRF token obtained` + ); + } catch (error) { + await alertService.error( + `Failed to initialize session for facility ${this.config.facilityId}: ${error}`, + error as Error + ); + throw error; + } } - throw error; - } -}; -async function collectJailData( - countyId: string, - source: string -): Promise { - // This is a stub - replace with actual data collection logic - console.log(`Collecting data for county: ${countyId}, source: ${source}`); - - // Mock data for demonstration - const now = new Date(); - const today = now.toISOString().split("T")[0]; - const timestamp = now.toISOString(); - - return [ - { - detaineeId: `${countyId}-${Date.now()}-001`, - timestamp, - status: "ACTIVE", - createdDate: today, - countyId, - source, - firstName: "John", - lastName: "Doe", - bookingDate: today, - charges: ["DWI", "Traffic Violation"], - ttl: Math.floor(Date.now() / 1000) + 365 * 24 * 60 * 60, // 1 year TTL - }, - ]; -} + // Collect all inmate data with pagination + public async collectAllInmates(): Promise { + const requestId = uuidv4(); + let skip = 0; + let totalProcessed = 0; + let batchNumber = 1; + let totalRecords: number | null = null; + + try { + await this.initializeSession(); -async function storeDetentionRecord(record: DetentionRecord): Promise { - const params = { - TableName: process.env.JAILDATA_TABLE!, - Item: record, - }; + while (true) { + const requestBody: InmateApiRequestBody = { + FilterOptionsParameters: { + IntersectionSearch: true, + SearchText: "", + Parameters: [], + }, + IncludeCount: true, + PagingOptions: { + SortOptions: [ + { + Name: "ArrestDate", + SortDirection: "Descending", + Sequence: 1, + }, + ], + Take: this.config.batchSize, + Skip: skip, + }, + }; - await docClient.send(new PutCommand(params)); - console.log(`Stored record for detainee: ${record.detaineeId}`); + const response = + await this.axiosInstance.post( + `/api/inmates/${this.facilityApiId}`, + requestBody + ); + + const batch = response.data; + + // Set total on first batch + if (totalRecords === null) { + totalRecords = batch.Total; + await alertService.info( + `Starting data collection for facility ${this.config.facilityId}: ${totalRecords} total records` + ); + } + + // Send batch to processing queue + await this.sendBatchToQueue({ + facilityId: this.config.facilityId, + batch, + batchNumber, + totalBatches: Math.ceil( + (totalRecords || 0) / this.config.batchSize + ), + requestId, + }); + + totalProcessed += batch.Inmates.length; + + // Check if we've processed all records + if ( + batch.Inmates.length < this.config.batchSize || + totalProcessed >= (totalRecords || 0) + ) { + break; + } + + skip += this.config.batchSize; + batchNumber++; + } + + await alertService.info( + `Data collection completed for facility ${this.config.facilityId}: ${totalProcessed} records in ${batchNumber} batches` + ); + } catch (error) { + await alertService.error( + `Data collection failed for facility ${this.config.facilityId}: ${error}`, + error as Error + ); + throw error; + } + } + + private async sendBatchToQueue( + message: BatchProcessingMessage + ): Promise { + const queueUrl = `https://sqs.${process.env.AWS_REGION}.amazonaws.com/${process.env.AWS_ACCOUNT_ID}/jaildata-batch-processing-${process.env.STAGE}`; + + const command = new SendMessageCommand({ + QueueUrl: queueUrl, + MessageBody: JSON.stringify(message), + MessageAttributes: { + facilityId: { + DataType: "String", + StringValue: message.facilityId, + }, + requestId: { + DataType: "String", + StringValue: message.requestId, + }, + batchNumber: { + DataType: "Number", + StringValue: message.batchNumber.toString(), + }, + }, + }); + + await sqsClient.send(command); + } } + +// Lambda handler for manual trigger (API Gateway) +export const collect = async ( + event: APIGatewayProxyEvent +): Promise => { + try { + // Load facility API IDs from Parameter Store + await FacilityMapper.loadApiIds(); + + const body = event.body ? JSON.parse(event.body) : {}; + const facilityId = body.facilityId || event.pathParameters?.facilityId; + + if (!facilityId) { + return { + statusCode: 400, + body: JSON.stringify({ error: "facilityId is required" }), + }; + } + + // Get base URL from SSM + const baseUrlParam = await ssmClient.send( + new GetParameterCommand({ + Name: "/jaildata/base-url", + WithDecryption: true, + }) + ); + + if (!baseUrlParam.Parameter?.Value) { + throw new Error("Base URL not configured in SSM Parameter Store"); + } + + const service = new DataCollectionService( + facilityId, + baseUrlParam.Parameter.Value, + 100 + ); + + // Initialize facility configuration + service.initializeFacility(); + + // Start collection (runs asynchronously) + service.collectAllInmates().catch((error) => { + alertService.error( + `Async data collection failed: ${error}`, + error as Error + ); + }); + + return { + statusCode: 202, + body: JSON.stringify({ + message: "Data collection started", + facilityId, + timestamp: new Date().toISOString(), + }), + }; + } catch (error) { + await alertService.error( + `Manual data collection failed: ${error}`, + error as Error + ); + + return { + statusCode: 500, + body: JSON.stringify({ + error: "Failed to start data collection", + message: (error as Error).message || "Unknown error", + }), + }; + } +}; + +// Lambda handler for scheduled trigger (EventBridge/CloudWatch Events) +export const collectScheduled = async ( + event: ScheduledEvent +): Promise => { + try { + // Load facility API IDs from Parameter Store + await FacilityMapper.loadApiIds(); + + // Get facilityId from event input + const eventInput = event.detail || {}; + const facilityId = eventInput.facilityId; + + if (!facilityId) { + throw new Error("facilityId is required in event input"); + } + + // Get base URL from SSM + const baseUrlParam = await ssmClient.send( + new GetParameterCommand({ + Name: "/jaildata/base-url", + WithDecryption: true, + }) + ); + + if (!baseUrlParam.Parameter?.Value) { + throw new Error("Base URL not configured in SSM Parameter Store"); + } + + const service = new DataCollectionService( + facilityId, + baseUrlParam.Parameter.Value, + 100 + ); + + // Initialize facility configuration + service.initializeFacility(); + + await service.collectAllInmates(); + + await alertService.info( + `Scheduled data collection completed successfully for facility ${facilityId}` + ); + } catch (error) { + await alertService.error( + `Scheduled data collection failed: ${error}`, + error as Error + ); + throw error; + } +}; diff --git a/serverless/api/handlers/detainee.ts b/serverless/api/handlers/detainee.ts index c32102a..04bdde7 100644 --- a/serverless/api/handlers/detainee.ts +++ b/serverless/api/handlers/detainee.ts @@ -1,113 +1,220 @@ import { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda"; -import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; -import { - DynamoDBDocumentClient, - QueryCommand, -} from "@aws-sdk/lib-dynamodb"; +import StorageClient from "../../lib/StorageClient"; +import AlertService, { AlertCategory } from "../../lib/AlertService"; +import { FacilityMapper } from "../../lib/FacilityMapping"; -const dynamoClient = new DynamoDBClient({ region: process.env.AWS_REGION }); -const docClient = DynamoDBDocumentClient.from(dynamoClient); +const alertService = AlertService.forCategory(AlertCategory.DATABASE); -export const get = async ( - event: APIGatewayProxyEvent +export const getInmatesByFacility = async ( + event: APIGatewayProxyEvent ): Promise => { - try { - const detaineeId = event.pathParameters?.detaineeId; - - if (!detaineeId) { - return { - statusCode: 400, - body: JSON.stringify({ error: "Missing detaineeId parameter" }), - }; + try { + const facilityId = event.pathParameters?.facilityId; + const limit = event.queryStringParameters?.limit + ? parseInt(event.queryStringParameters.limit) + : 100; + + if (!facilityId) { + return { + statusCode: 400, + body: JSON.stringify({ error: "Missing facilityId parameter" }), + }; + } + + // Validate facility exists in our mapping + if (!FacilityMapper.isValidFacilityName(facilityId)) { + return { + statusCode: 400, + body: JSON.stringify({ + error: `Invalid facilityId: ${facilityId}. Valid facilities: ${FacilityMapper.getAllFacilities() + .map((f) => f.name) + .join(", ")}`, + }), + }; + } + + const inmates = await StorageClient.getInmatesByFacility( + facilityId, + limit + ); + + return { + statusCode: 200, + body: JSON.stringify({ + facilityId, + inmates, + count: inmates.length, + description: "Recent inmates (last 24 hours)", + timestamp: new Date().toISOString(), + }), + }; + } catch (error) { + await alertService.error( + "Error getting inmates by facility", + error as Error + ); + return { + statusCode: 500, + body: JSON.stringify({ error: "Internal server error" }), + }; } +}; - const params = { - TableName: process.env.JAILDATA_TABLE!, - KeyConditionExpression: "detaineeId = :detaineeId", - ExpressionAttributeValues: { - ":detaineeId": detaineeId, - }, - ScanIndexForward: false, // Most recent first - Limit: 10, // Get last 10 records - }; - - const result = await docClient.send(new QueryCommand(params)); - - if (!result.Items || result.Items.length === 0) { - return { - statusCode: 404, - body: JSON.stringify({ error: "Detainee not found" }), - }; +export const getAllActiveInmates = async ( + event: APIGatewayProxyEvent +): Promise => { + try { + const limit = event.queryStringParameters?.limit + ? parseInt(event.queryStringParameters.limit) + : 100; + + const inmates = await StorageClient.getAllRecentInmates(limit); + + return { + statusCode: 200, + body: JSON.stringify({ + inmates, + count: inmates.length, + description: "Recent inmates (last 24 hours)", + timestamp: new Date().toISOString(), + }), + }; + } catch (error) { + await alertService.error( + "Error getting all active inmates", + error as Error + ); + return { + statusCode: 500, + body: JSON.stringify({ error: "Internal server error" }), + }; } +}; + +export const searchInmatesByName = async ( + event: APIGatewayProxyEvent +): Promise => { + try { + const facilityId = event.pathParameters?.facilityId; + const lastName = event.queryStringParameters?.lastName; + const limit = event.queryStringParameters?.limit + ? parseInt(event.queryStringParameters.limit) + : 50; - return { - statusCode: 200, - body: JSON.stringify({ - detaineeId, - records: result.Items, - count: result.Items.length, - }), - }; - } catch (error) { - console.error("Error getting detainee:", error); - return { - statusCode: 500, - body: JSON.stringify({ error: "Internal server error" }), - }; - } + if (!facilityId) { + return { + statusCode: 400, + body: JSON.stringify({ error: "Missing facilityId parameter" }), + }; + } + + // Validate facility exists in our mapping + if (!FacilityMapper.isValidFacilityName(facilityId)) { + return { + statusCode: 400, + body: JSON.stringify({ + error: `Invalid facilityId: ${facilityId}. Valid facilities: ${FacilityMapper.getAllFacilities() + .map((f) => f.name) + .join(", ")}`, + }), + }; + } + + if (!lastName) { + return { + statusCode: 400, + body: JSON.stringify({ + error: "Missing lastName query parameter", + }), + }; + } + + const inmates = await StorageClient.searchInmatesByLastName( + facilityId, + lastName, + limit + ); + + return { + statusCode: 200, + body: JSON.stringify({ + facilityId, + searchTerm: lastName, + inmates, + count: inmates.length, + timestamp: new Date().toISOString(), + }), + }; + } catch (error) { + await alertService.error( + "Error searching inmates by name", + error as Error + ); + return { + statusCode: 500, + body: JSON.stringify({ error: "Internal server error" }), + }; + } }; -export const listActive = async ( - event: APIGatewayProxyEvent +export const getSpecificInmate = async ( + event: APIGatewayProxyEvent ): Promise => { - try { - const limit = event.queryStringParameters?.limit - ? parseInt(event.queryStringParameters.limit) - : 100; - const daysBack = event.queryStringParameters?.days - ? parseInt(event.queryStringParameters.days) - : 7; - - // Calculate date range - const endDate = new Date(); - const startDate = new Date( - endDate.getTime() - daysBack * 24 * 60 * 60 * 1000 - ); - const startDateStr = startDate.toISOString().split("T")[0]; - - const params = { - TableName: process.env.JAILDATA_TABLE!, - IndexName: "StatusCreatedDateIndex", - KeyConditionExpression: "#status = :status AND createdDate >= :startDate", - ExpressionAttributeNames: { - "#status": "status", - }, - ExpressionAttributeValues: { - ":status": "ACTIVE", - ":startDate": startDateStr, - }, - ScanIndexForward: false, // Most recent first - Limit: limit, - }; - - const result = await docClient.send(new QueryCommand(params)); - - return { - statusCode: 200, - body: JSON.stringify({ - activeDetainees: result.Items || [], - count: result.Items?.length || 0, - dateRange: { - start: startDateStr, - end: endDate.toISOString().split("T")[0], - }, - }), - }; - } catch (error) { - console.error("Error listing active detainees:", error); - return { - statusCode: 500, - body: JSON.stringify({ error: "Internal server error" }), - }; - } + try { + const facilityId = event.pathParameters?.facilityId; + const lastName = event.pathParameters?.lastName; + const firstName = event.pathParameters?.firstName; + const middleName = event.pathParameters?.middleName || ""; + const recordDate = event.pathParameters?.recordDate; + + if (!facilityId || !lastName || !firstName || !recordDate) { + return { + statusCode: 400, + body: JSON.stringify({ + error: "Missing required parameters: facilityId, lastName, firstName, recordDate", + }), + }; + } + + // Validate facility exists in our mapping + if (!FacilityMapper.isValidFacilityName(facilityId)) { + return { + statusCode: 400, + body: JSON.stringify({ + error: `Invalid facilityId: ${facilityId}. Valid facilities: ${FacilityMapper.getAllFacilities() + .map((f) => f.name) + .join(", ")}`, + }), + }; + } + + const inmate = await StorageClient.getInmate( + facilityId, + lastName, + firstName, + middleName, + recordDate + ); + + if (!inmate) { + return { + statusCode: 404, + body: JSON.stringify({ error: "Inmate not found" }), + }; + } + + return { + statusCode: 200, + body: JSON.stringify(inmate), + }; + } catch (error) { + await alertService.error( + "Error getting specific inmate", + error as Error + ); + return { + statusCode: 500, + body: JSON.stringify({ error: "Internal server error" }), + }; + } }; diff --git a/serverless/api/handlers/status.ts b/serverless/api/handlers/status.ts index dfdf55b..d1fd867 100644 --- a/serverless/api/handlers/status.ts +++ b/serverless/api/handlers/status.ts @@ -1,32 +1,32 @@ import { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda"; export const get = async ( - _event: APIGatewayProxyEvent + _event: APIGatewayProxyEvent, ): Promise => { - try { - // Basic health check - const status = { - service: "detention-data-api", - status: "healthy", - timestamp: new Date().toISOString(), - environment: process.env.STAGE || "unknown", - version: "1.0.0", - }; + try { + // Basic health check + const status = { + service: "detention-data-api", + status: "healthy", + timestamp: new Date().toISOString(), + environment: process.env.STAGE || "unknown", + version: "1.0.0", + }; - return { - statusCode: 200, - body: JSON.stringify(status), - }; - } catch (error) { - console.error("Error in status check:", error); - return { - statusCode: 500, - body: JSON.stringify({ - service: "jaildata-api", - status: "unhealthy", - error: "Internal server error", - timestamp: new Date().toISOString(), - }), - }; - } + return { + statusCode: 200, + body: JSON.stringify(status), + }; + } catch (error) { + console.error("Error in status check:", error); + return { + statusCode: 500, + body: JSON.stringify({ + service: "jaildata-api", + status: "unhealthy", + error: "Internal server error", + timestamp: new Date().toISOString(), + }), + }; + } }; diff --git a/serverless/api/serverless.yml b/serverless/api/serverless.yml index 0dea416..9e803a0 100644 --- a/serverless/api/serverless.yml +++ b/serverless/api/serverless.yml @@ -13,12 +13,15 @@ provider: JAILDATA_TABLE: !Ref JailDataTable ERROR_CACHE_TABLE: !Ref ErrorCacheTable SERVICE_NAME: ${self:service} + AWS_ACCOUNT_ID: !Ref "AWS::AccountId" + BATCH_PROCESSING_QUEUE_URL: !Ref BatchProcessingQueue iam: role: statements: - Effect: Allow Action: - dynamodb:BatchGetItem + - dynamodb:BatchWriteItem - dynamodb:GetItem - dynamodb:PutItem - dynamodb:Query @@ -53,6 +56,16 @@ provider: Resource: - !Ref AlertTopic + - Effect: Allow + Action: + - sqs:SendMessage + - sqs:ReceiveMessage + - sqs:DeleteMessage + - sqs:GetQueueAttributes + Resource: + - !GetAtt BatchProcessingQueue.Arn + - !GetAtt BatchProcessingDeadLetterQueue.Arn + apiGateway: disableDefaultEndpoint: true apiKeys: @@ -124,40 +137,32 @@ resources: TopicArn: !Ref AlertTopic Endpoint: ${ssm:/jaildata/alert-email} - # DynamoDB table for jail data + # DynamoDB table for jail data (single table design) JailDataTable: Type: AWS::DynamoDB::Table Properties: TableName: jaildata-${self:provider.stage} BillingMode: PAY_PER_REQUEST AttributeDefinitions: - - AttributeName: detaineeId + - AttributeName: PK AttributeType: S - - AttributeName: timestamp + - AttributeName: SK AttributeType: S - - AttributeName: status + - AttributeName: GSI1PK AttributeType: S - - AttributeName: createdDate + - AttributeName: GSI1SK AttributeType: S KeySchema: - - AttributeName: detaineeId + - AttributeName: PK KeyType: HASH - - AttributeName: timestamp + - AttributeName: SK KeyType: RANGE GlobalSecondaryIndexes: - - IndexName: StatusCreatedDateIndex - KeySchema: - - AttributeName: status - KeyType: HASH - - AttributeName: createdDate - KeyType: RANGE - Projection: - ProjectionType: ALL - - IndexName: CreatedDateTimestampIndex + - IndexName: GSI1 KeySchema: - - AttributeName: createdDate + - AttributeName: GSI1PK KeyType: HASH - - AttributeName: timestamp + - AttributeName: GSI1SK KeyType: RANGE Projection: ProjectionType: ALL @@ -169,6 +174,38 @@ resources: - Key: Service Value: JailData + # SQS queue for batch processing (handles inmate data batches) + BatchProcessingQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: jaildata-batch-processing-${self:provider.stage} + VisibilityTimeoutSeconds: 300 + MessageRetentionPeriod: 1209600 # 14 days + RedrivePolicy: + deadLetterTargetArn: !GetAtt BatchProcessingDeadLetterQueue.Arn + maxReceiveCount: 3 + Tags: + - Key: Name + Value: JailData Batch Processing Queue + - Key: Environment + Value: ${self:provider.stage} + - Key: Service + Value: JailData + + # Dead letter queue for batch processing + BatchProcessingDeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: jaildata-batch-processing-dlq-${self:provider.stage} + MessageRetentionPeriod: 1209600 # 14 days + Tags: + - Key: Name + Value: JailData Batch Processing Dead Letter Queue + - Key: Environment + Value: ${self:provider.stage} + - Key: Service + Value: JailData + # DynamoDB table for error cache and deduplication ErrorCacheTable: Type: AWS::DynamoDB::Table @@ -213,37 +250,151 @@ resources: Name: ${self:service}-${self:provider.stage}-CommunityUsagePlanId functions: - # Scheduled data collection function - dataCollection: - handler: handlers/data-collection.execute + # Manual data collection trigger + dataCollectionManual: + handler: handlers/data-collection.collect + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - http: + path: /collect/{facilityId} + method: post + private: true + + # Scheduled data collection functions for different facilities + # To add a new facility: copy a function, change the name, set the facility ID in the input, and pick a unique schedule + + # Wake County - 10:00 AM UTC daily + dataCollectionWake: + handler: handlers/data-collection.collectScheduled memorySize: 1024 timeout: 600 # 10 minutes events: - # Default collection at 10 AM UTC - schedule: rate: cron(0 10 * * ? *) - input: '{"countyId": "wake", "source": "wake-county"}' - # Additional configurable collections can be added here - # Example: 10:30 AM UTC for Mecklenburg County - # - schedule: - # rate: cron(30 10 * * ? *) - # input: '{"countyId": "mecklenburg", "source": "mecklenburg-county"}' - - # Get detainee data - getDetainee: - handler: handlers/detainee.get + enabled: true + name: jaildata-collection-wake-${self:provider.stage} + description: "Daily Wake County jail data collection" + input: '{"facilityId": "wake"}' + + # Buncombe County - 10:15 AM UTC daily + dataCollectionBuncombe: + handler: handlers/data-collection.collectScheduled + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - schedule: + rate: cron(15 10 * * ? *) + enabled: true + name: jaildata-collection-buncombe-${self:provider.stage} + description: "Daily Buncombe County jail data collection" + input: '{"facilityId": "buncombe"}' + + # DISABLED: Mecklenburg County - Need API ID (currently 0) + dataCollectionMecklenburg: + handler: handlers/data-collection.collectScheduled + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - schedule: + rate: cron(30 10 * * ? *) + enabled: false # DISABLED: Need API ID + name: jaildata-collection-mecklenburg-${self:provider.stage} + description: "Daily Mecklenburg County jail data collection" + input: '{"facilityId": "mecklenburg"}' + + # DISABLED: Durham County - Need API ID (currently 0) + dataCollectionDurham: + handler: handlers/data-collection.collectScheduled + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - schedule: + rate: cron(0 11 * * ? *) + enabled: false # DISABLED: Need API ID + name: jaildata-collection-durham-${self:provider.stage} + description: "Daily Durham County jail data collection" + input: '{"facilityId": "durham"}' + + # DISABLED: Orange County - Need API ID (currently 0) + dataCollectionOrange: + handler: handlers/data-collection.collectScheduled + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - schedule: + rate: cron(30 11 * * ? *) + enabled: false # DISABLED: Need API ID + name: jaildata-collection-orange-${self:provider.stage} + description: "Daily Orange County jail data collection" + input: '{"facilityId": "orange"}' + + # DISABLED: Guilford County - Need API ID (currently 0) + dataCollectionGuilford: + handler: handlers/data-collection.collectScheduled + memorySize: 1024 + timeout: 600 # 10 minutes + events: + - schedule: + rate: cron(0 12 * * ? *) + enabled: false # DISABLED: Need API ID + name: jaildata-collection-guilford-${self:provider.stage} + description: "Daily Guilford County jail data collection" + input: '{"facilityId": "guilford"}' + + # Batch processing function (triggered by SQS) + batchProcessing: + handler: handlers/batch-processing.processBatch + memorySize: 512 + timeout: 300 # 5 minutes + events: + - sqs: + arn: !GetAtt BatchProcessingQueue.Arn + batchSize: 10 + maximumBatchingWindowInSeconds: 30 + + # Get inmates by facility (with status filter) + getInmatesByFacility: + handler: handlers/detainee.getInmatesByFacility + events: + - http: + path: inmates/{facilityId} + method: get + private: true + + # Get all active inmates across facilities + getAllActiveInmates: + handler: handlers/detainee.getAllActiveInmates + events: + - http: + path: inmates/active + method: get + private: true + + # Search inmates by last name within a facility + searchInmatesByName: + handler: handlers/detainee.searchInmatesByName + events: + - http: + path: inmates/{facilityId}/search + method: get + private: true + + # Get specific inmate record + getSpecificInmate: + handler: handlers/detainee.getSpecificInmate events: - http: - path: detainee/{detaineeId} + path: inmates/{facilityId}/{lastName}/{firstName}/{recordDate} method: get private: true - # List active detainees - listActiveDetainees: - handler: handlers/detainee.listActive + # Get specific inmate record with middle name + getSpecificInmateWithMiddle: + handler: handlers/detainee.getSpecificInmate events: - http: - path: detainees/active + path: inmates/{facilityId}/{lastName}/{firstName}/{middleName}/{recordDate} method: get private: true diff --git a/serverless/eslint.config.js b/serverless/eslint.config.js index d8486c8..806acbe 100644 --- a/serverless/eslint.config.js +++ b/serverless/eslint.config.js @@ -26,11 +26,14 @@ module.exports = tseslint.config( }, rules: { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": ["error", { - "argsIgnorePattern": "^_", - "varsIgnorePattern": "^_", - "ignoreRestSiblings": true - }], + "@typescript-eslint/no-unused-vars": [ + "error", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + ignoreRestSiblings: true, + }, + ], "max-len": [ "error", { @@ -67,5 +70,5 @@ module.exports = tseslint.config( rules: { "@typescript-eslint/no-require-imports": "off", }, - } + }, ); diff --git a/serverless/jest.config.js b/serverless/jest.config.js index c77a84f..8a3c3bd 100644 --- a/serverless/jest.config.js +++ b/serverless/jest.config.js @@ -1,17 +1,17 @@ module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - roots: ['/lib', '/api'], - testMatch: ['**/__tests__/**/*.test.ts'], + preset: "ts-jest", + testEnvironment: "node", + roots: ["/lib", "/api"], + testMatch: ["**/__tests__/**/*.test.ts"], transform: { - '^.+\\.ts$': 'ts-jest', + "^.+\\.ts$": "ts-jest", }, collectCoverageFrom: [ - 'lib/**/*.ts', - 'api/**/*.ts', - '!**/*.d.ts', - '!**/node_modules/**', + "lib/**/*.ts", + "api/**/*.ts", + "!**/*.d.ts", + "!**/node_modules/**", ], - coverageReporters: ['text', 'lcov', 'html'], - moduleFileExtensions: ['ts', 'js', 'json'], -}; \ No newline at end of file + coverageReporters: ["text", "lcov", "html"], + moduleFileExtensions: ["ts", "js", "json"], +}; diff --git a/serverless/lib/AlertService.ts b/serverless/lib/AlertService.ts index f95c7e3..7e82b4d 100644 --- a/serverless/lib/AlertService.ts +++ b/serverless/lib/AlertService.ts @@ -46,6 +46,8 @@ export enum AlertCategory { PORTAL = "PORTAL", QUEUE = "QUEUE", SYSTEM = "SYS", + DATA_COLLECTION = "DATA_COLLECTION", + BATCH_PROCESSING = "BATCH_PROCESSING", } // Error context to provide additional information diff --git a/serverless/lib/FacilityMapping.ts b/serverless/lib/FacilityMapping.ts new file mode 100644 index 0000000..a5ce98f --- /dev/null +++ b/serverless/lib/FacilityMapping.ts @@ -0,0 +1,203 @@ +/** + * Facility mapping between human-friendly names and numeric API IDs + * API IDs are loaded from Parameter Store + * + * Parameter Store Convention: + * - API IDs are stored at: /jaildata/facilities/{facility-name}/api-id + * - Where {facility-name} corresponds to the 'name' field in the facility config + * - Example: /jaildata/facilities/buncombe/api-id + */ + +import { SSMClient, GetParameterCommand } from "@aws-sdk/client-ssm"; + +export interface FacilityConfig { + /** Human-friendly name used in our APIs */ + name: string; + /** Numeric facility ID used by the jail data API */ + apiId: number; + /** Full county name for display purposes */ + displayName: string; +} + +interface BaseFacilityConfig { + /** Human-friendly name used in our APIs */ + name: string; + /** Full county name for display purposes */ + displayName: string; +} + +// Parameter Store path convention: /jaildata/facilities/{facility-name}/api-id +const FACILITY_API_ID_PARAMETER_PREFIX = "/jaildata/facilities"; + +// Base facility configuration without sensitive API IDs +const BASE_FACILITY_MAPPING: Record = { + // wake: { + // name: 'wake', + // displayName: 'Wake County' + // }, + buncombe: { + name: "buncombe", + displayName: "Buncombe County", + }, +}; + +// Static facility mapping (initialized with API ID 0, updated via loadApiIds) +export const FACILITY_MAPPING: Record = + Object.fromEntries( + Object.entries(BASE_FACILITY_MAPPING).map(([key, baseConfig]) => [ + key, + { + name: baseConfig.name, + displayName: baseConfig.displayName, + apiId: 0, // Will be loaded from Parameter Store + }, + ]) + ); + +// Reverse mapping for API ID to name lookups (will be populated after loading API IDs) +export const API_ID_TO_NAME: Record = {}; + +// Singleton class to manage loading API IDs from Parameter Store +class FacilityApiLoader { + private static instance: FacilityApiLoader; + private ssmClient: SSMClient; + private loaded = false; + + private constructor() { + this.ssmClient = new SSMClient({ region: process.env.AWS_REGION }); + } + + static getInstance(): FacilityApiLoader { + if (!FacilityApiLoader.instance) { + FacilityApiLoader.instance = new FacilityApiLoader(); + } + return FacilityApiLoader.instance; + } + + /** + * Load API IDs from Parameter Store and update the facility mapping + * This should be called once during application initialization + */ + async loadApiIds(): Promise { + if (this.loaded) { + return; + } + + console.log("Loading facility API IDs from Parameter Store..."); + + for (const [facilityName, baseConfig] of Object.entries( + BASE_FACILITY_MAPPING + )) { + try { + // Use convention-based parameter name: /jaildata/facilities/{name}/api-id + const parameterName = `${FACILITY_API_ID_PARAMETER_PREFIX}/${baseConfig.name}/api-id`; + + const command = new GetParameterCommand({ + Name: parameterName, + WithDecryption: true, + }); + + const response = await this.ssmClient.send(command); + const apiId = response.Parameter?.Value + ? parseInt(response.Parameter.Value, 10) + : 0; + + // Update the facility mapping + FACILITY_MAPPING[facilityName].apiId = apiId; + + // Build reverse mapping for active facilities + if (apiId > 0) { + API_ID_TO_NAME[apiId] = facilityName; + } + + console.log( + `Loaded API ID for ${facilityName}: ${ + apiId > 0 ? "[CONFIGURED]" : "[NOT CONFIGURED]" + }` + ); + } catch (error) { + console.warn( + `Failed to load API ID for facility ${facilityName}:`, + error + ); + // Keep API ID as 0 if parameter not found + } + } + + this.loaded = true; + console.log("Facility API IDs loaded successfully"); + } + + isLoaded(): boolean { + return this.loaded; + } +} + +export class FacilityMapper { + /** + * Load API IDs from Parameter Store (call this during app initialization) + */ + static async loadApiIds(): Promise { + const loader = FacilityApiLoader.getInstance(); + await loader.loadApiIds(); + } + + /** + * Check if API IDs have been loaded from Parameter Store + */ + static isApiIdsLoaded(): boolean { + const loader = FacilityApiLoader.getInstance(); + return loader.isLoaded(); + } + + /** + * Get facility config by human-friendly name + */ + static getFacilityByName(name: string): FacilityConfig | null { + return FACILITY_MAPPING[name.toLowerCase()] || null; + } + + /** + * Get facility config by numeric API ID + */ + static getFacilityByApiId(apiId: number): FacilityConfig | null { + const name = API_ID_TO_NAME[apiId]; + return name ? FACILITY_MAPPING[name] : null; + } + + /** + * Get the numeric API ID for a facility name + */ + static getApiIdForFacility(name: string): number | null { + const facility = this.getFacilityByName(name); + return facility ? facility.apiId : null; + } + + /** + * Get human-friendly name for an API ID + */ + static getNameForApiId(apiId: number): string | null { + return API_ID_TO_NAME[apiId] || null; + } + + /** + * Validate that a facility name exists in our mapping + */ + static isValidFacilityName(name: string): boolean { + return name.toLowerCase() in FACILITY_MAPPING; + } + + /** + * Get all configured facilities + */ + static getAllFacilities(): FacilityConfig[] { + return Object.values(FACILITY_MAPPING); + } + + /** + * Get all facilities that have valid API IDs (> 0) + */ + static getActiveFacilities(): FacilityConfig[] { + return Object.values(FACILITY_MAPPING).filter((f) => f.apiId > 0); + } +} diff --git a/serverless/lib/StorageClient.ts b/serverless/lib/StorageClient.ts new file mode 100644 index 0000000..3bebd23 --- /dev/null +++ b/serverless/lib/StorageClient.ts @@ -0,0 +1,459 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, + BatchWriteCommand, +} from "@aws-sdk/lib-dynamodb"; +import { InmateDynamoRecord, InmateRecord } from "./types"; + +// DynamoDB-specific attributes that should be removed from API responses +const DYNAMO_ATTRIBUTES = ["PK", "SK", "GSI1PK", "GSI1SK"]; + +/** + * Removes DynamoDB-specific attributes from an object or array of objects + * @param data The data to clean + * @returns Cleaned data without DynamoDB attributes + */ +function removeDynamoAttributes(data: T): T { + if (!data) { + return data; + } + + if (Array.isArray(data)) { + return data.map(removeDynamoAttributes) as unknown as T; + } + + if (typeof data === "object" && data !== null) { + const cleanedObject = { ...(data as object) } as Record< + string, + unknown + >; + + // Remove DynamoDB attributes + DYNAMO_ATTRIBUTES.forEach((attr) => { + if (attr in cleanedObject) { + delete cleanedObject[attr]; + } + }); + + // Recursively clean nested objects and arrays + Object.keys(cleanedObject).forEach((key) => { + cleanedObject[key] = removeDynamoAttributes(cleanedObject[key]); + }); + + return cleanedObject as T; + } + + return data; +} + +export interface DynamoCompositeKey { + PK: string; + SK: string; +} + +// Key helper for generating consistent partition and sort keys +export const Key = { + Inmate: ( + facilityId: string, + lastName: string, + firstName: string, + middleName: string, + recordDate: string + ) => ({ + PK: `INMATE#${facilityId}#${cleanNameForKey( + lastName + )}#${cleanNameForKey(firstName)}#${cleanNameForKey(middleName)}`, + SK: recordDate, // Format: M/d/yyyy + }), + + ErrorCache: (errorKey: string) => ({ + PK: "ERROR_CACHE", + SK: errorKey, + }), +}; + +/** + * Clean name for use in DynamoDB keys (remove special characters, lowercase) + */ +function cleanNameForKey(name?: string): string { + return (name || "").toLowerCase().replace(/[^a-z0-9]/g, ""); +} + +/** + * Determine if an inmate is active based on last updated time + */ +/** + * Generate GSI1 keys for facility-based queries + */ +function getFacilityGSIKeys(facilityId: string, recordDate: string) { + return { + GSI1PK: `FACILITY#${facilityId}`, + GSI1SK: recordDate, // YYYY-MM-DD format for sorting + }; +} + +const ddbClient = new DynamoDBClient({ + region: process.env.AWS_REGION || "us-east-2", +}); +const dynamoDb = DynamoDBDocumentClient.from(ddbClient); + +const TABLE_NAME = + process.env.JAILDATA_TABLE || `jaildata-${process.env.STAGE || "dev"}`; + +/** + * Get a single item from DynamoDB + */ +async function get(key: DynamoCompositeKey): Promise { + try { + const result = await dynamoDb.send( + new GetCommand({ + TableName: TABLE_NAME, + Key: key, + }) + ); + + if (!result.Item) { + return null; + } + + return removeDynamoAttributes(result.Item) as T; + } catch (error) { + console.error("Error getting item from DynamoDB:", error); + throw error; + } +} + +/** + * Save a single item to DynamoDB + */ +async function save>( + key: DynamoCompositeKey, + item: T +): Promise { + try { + const itemToSave = { + ...key, + ...item, + lastUpdated: new Date().toISOString(), + }; + + await dynamoDb.send( + new PutCommand({ + TableName: TABLE_NAME, + Item: itemToSave, + }) + ); + } catch (error) { + console.error("Error saving item to DynamoDB:", error); + throw error; + } +} + +/** + * Parse arrest date from various formats and return M/d/yyyy format + */ +function parseArrestDate(arrestDate?: string): string { + const now = new Date(); + + if (!arrestDate) { + // Use current date if no arrest date + return now.toISOString().split("T")[0]; // YYYY-MM-DD + } + + try { + // Handle format: M/d/yyyy h:mm:ss [AP]M + const dateMatch = arrestDate.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})/); + if (dateMatch) { + const month = dateMatch[1].padStart(2, "0"); + const day = dateMatch[2].padStart(2, "0"); + const year = dateMatch[3]; + return `${year}-${month}-${day}`; + } + + // Try to parse as ISO date + const parsedDate = new Date(arrestDate); + if (!isNaN(parsedDate.getTime())) { + return parsedDate.toISOString().split("T")[0]; // YYYY-MM-DD + } + } catch (error) { + console.warn(`Could not parse arrest date: ${arrestDate}`, error); + } + + // Fallback to current date + return `${now.getMonth() + 1}/${now.getDate()}/${now.getFullYear()}`; +} + +const StorageClient = { + /** + * Save or update an inmate record + */ + async saveInmate(facilityId: string, inmate: InmateRecord): Promise { + try { + // Extract key information + const firstName = inmate.FirstName || ""; + const lastName = inmate.LastName || ""; + const middleName = inmate.MiddleName || ""; + const totalBondAmount = + typeof inmate.TotalBondAmount === "string" + ? parseFloat(inmate.TotalBondAmount) || 0 + : inmate.TotalBondAmount || 0; + + // Parse arrest date to get the record date + const recordDate = parseArrestDate(inmate.ArrestDate); + const lastUpdated = new Date().toISOString(); + + // Generate deterministic PK based on identity + const inmateKey = Key.Inmate( + facilityId, + lastName, + firstName, + middleName, + recordDate + ); + const facilityGSI = getFacilityGSIKeys(facilityId, recordDate); + + const dynamoRecord: InmateDynamoRecord = { + ...inmateKey, + ...facilityGSI, + totalBondAmount, + recordDate, + lastUpdated, + rawData: inmate, + }; + + await save( + { PK: dynamoRecord.PK, SK: dynamoRecord.SK }, + dynamoRecord + ); + } catch (error) { + console.error("Error saving inmate record:", error); + throw error; + } + }, + + /** + * Get inmate records by facility (defaults to recent records - within 24 hours) + */ + async getInmatesByFacility( + facilityId: string, + limit = 100 + ): Promise { + const oneDayAgo = new Date(); + oneDayAgo.setDate(oneDayAgo.getDate() - 1); + const cutoffDate = oneDayAgo.toISOString().split("T")[0]; // YYYY-MM-DD + + return this.getInmatesByFacilityAndDateRange( + facilityId, + cutoffDate, + undefined, + limit + ); + }, + + /** + * Get inmate records by facility and date range + */ + async getInmatesByFacilityAndDateRange( + facilityId: string, + startDate?: string, // YYYY-MM-DD format + endDate?: string, // YYYY-MM-DD format + limit = 100 + ): Promise { + try { + let keyConditionExpression = "GSI1PK = :gsi1pk"; + const expressionAttributeValues: Record = { + ":gsi1pk": `FACILITY#${facilityId}`, + }; + + // Add date range filtering if provided + if (startDate && endDate) { + keyConditionExpression += + " AND GSI1SK BETWEEN :startDate AND :endDate"; + expressionAttributeValues[":startDate"] = startDate; + expressionAttributeValues[":endDate"] = endDate; + } else if (startDate) { + keyConditionExpression += " AND GSI1SK >= :startDate"; + expressionAttributeValues[":startDate"] = startDate; + } else if (endDate) { + keyConditionExpression += " AND GSI1SK <= :endDate"; + expressionAttributeValues[":endDate"] = endDate; + } + + const result = await dynamoDb.send( + new QueryCommand({ + TableName: TABLE_NAME, + IndexName: "GSI1", + KeyConditionExpression: keyConditionExpression, + ExpressionAttributeValues: expressionAttributeValues, + ScanIndexForward: false, // Most recent first + Limit: limit, + }) + ); + + return (result.Items || []).map((item) => + removeDynamoAttributes(item)) as InmateDynamoRecord[]; + } catch (error) { + console.error( + "Error getting inmates by facility and date range:", + error + ); + throw error; + } + }, + + /** + * Get all active inmates across all facilities + */ + async getAllRecentInmates(limit = 100): Promise { + try { + // Get recent inmates from the last 24 hours across all facilities + const oneDayAgo = new Date(); + oneDayAgo.setDate(oneDayAgo.getDate() - 1); + const cutoffDate = oneDayAgo.toISOString().split("T")[0]; // YYYY-MM-DD + + const result = await dynamoDb.send( + new QueryCommand({ + TableName: TABLE_NAME, + IndexName: "GSI1", + KeyConditionExpression: + "begins_with(GSI1PK, :facilityPrefix) AND GSI1SK >= :cutoffDate", + ExpressionAttributeValues: { + ":facilityPrefix": "FACILITY#", + ":cutoffDate": cutoffDate, + }, + ScanIndexForward: false, // Most recent first + Limit: limit, + }) + ); + + return (result.Items || []).map((item) => + removeDynamoAttributes(item)) as InmateDynamoRecord[]; + } catch (error) { + console.error("Error getting all recent inmates:", error); + throw error; + } + }, + + /** + * Search inmates by last name within a facility + */ + async searchInmatesByLastName( + facilityId: string, + lastName: string, + limit = 50 + ): Promise { + try { + // Use begins_with on PK to find inmates with matching last name + const result = await dynamoDb.send( + new QueryCommand({ + TableName: TABLE_NAME, + KeyConditionExpression: "begins_with(PK, :pkPrefix)", + ExpressionAttributeValues: { + ":pkPrefix": `INMATE#${facilityId}#${lastName.toUpperCase()}`, + }, + ScanIndexForward: true, // Alphabetical order + Limit: limit, + }) + ); + + return (result.Items || []).map((item) => + removeDynamoAttributes(item)) as InmateDynamoRecord[]; + } catch (error) { + console.error("Error searching inmates by last name:", error); + throw error; + } + }, + + /** + * Get a specific inmate record by identity + */ + async getInmate( + facilityId: string, + lastName: string, + firstName: string, + middleName: string, + recordDate: string + ): Promise { + const key = Key.Inmate( + facilityId, + lastName, + firstName, + middleName, + recordDate + ); + return await get(key); + }, + + /** + * Batch save multiple inmate records (for performance) + */ + async batchSaveInmates( + facilityId: string, + inmates: InmateRecord[] + ): Promise { + try { + // Process inmates in chunks of 25 (DynamoDB BatchWrite limit) + const chunkSize = 25; + + for (let i = 0; i < inmates.length; i += chunkSize) { + const chunk = inmates.slice(i, i + chunkSize); + + const putRequests = chunk.map((inmate) => { + const firstName = inmate.FirstName || ""; + const lastName = inmate.LastName || ""; + const middleName = inmate.MiddleName || ""; + const totalBondAmount = + typeof inmate.TotalBondAmount === "string" + ? parseFloat(inmate.TotalBondAmount) || 0 + : inmate.TotalBondAmount || 0; + + const recordDate = parseArrestDate(inmate.ArrestDate); + const lastUpdated = new Date().toISOString(); + + const inmateKey = Key.Inmate( + facilityId, + lastName, + firstName, + middleName, + recordDate + ); + const facilityGSI = getFacilityGSIKeys( + facilityId, + recordDate + ); + + const dynamoRecord: InmateDynamoRecord = { + ...inmateKey, + ...facilityGSI, + totalBondAmount, + recordDate, + lastUpdated, + rawData: inmate, + }; + + return { + PutRequest: { + Item: dynamoRecord, + }, + }; + }); + + await dynamoDb.send( + new BatchWriteCommand({ + RequestItems: { + [TABLE_NAME]: putRequests, + }, + }) + ); + } + } catch (error) { + console.error("Error batch saving inmate records:", error); + throw error; + } + }, +}; + +export default StorageClient; diff --git a/serverless/lib/__tests__/AlertService.test.ts b/serverless/lib/__tests__/AlertService.test.ts index 990bc08..6956cec 100644 --- a/serverless/lib/__tests__/AlertService.test.ts +++ b/serverless/lib/__tests__/AlertService.test.ts @@ -200,6 +200,8 @@ describe("AlertService", () => { expect(AlertCategory.PORTAL).toBe("PORTAL"); expect(AlertCategory.QUEUE).toBe("QUEUE"); expect(AlertCategory.SYSTEM).toBe("SYS"); + expect(AlertCategory.DATA_COLLECTION).toBe("DATA_COLLECTION"); + expect(AlertCategory.BATCH_PROCESSING).toBe("BATCH_PROCESSING"); }); }); diff --git a/serverless/lib/__tests__/FacilityMapping.test.ts b/serverless/lib/__tests__/FacilityMapping.test.ts new file mode 100644 index 0000000..6d00e0e --- /dev/null +++ b/serverless/lib/__tests__/FacilityMapping.test.ts @@ -0,0 +1,171 @@ +import { + FacilityMapper, + FACILITY_MAPPING, + API_ID_TO_NAME, +} from "../FacilityMapping"; + +describe("FacilityMapping", () => { + describe("FACILITY_MAPPING", () => { + it("should have the correct structure for buncombe facility", () => { + expect(FACILITY_MAPPING.buncombe).toEqual({ + name: "buncombe", + apiId: 0, // Initially 0, loaded from Parameter Store + displayName: "Buncombe County", + }); + }); + + it("should only contain buncombe facility (wake is commented out)", () => { + expect(Object.keys(FACILITY_MAPPING)).toEqual(["buncombe"]); + }); + }); + + describe("API_ID_TO_NAME", () => { + it("should be empty initially (populated after loadApiIds)", () => { + expect(Object.keys(API_ID_TO_NAME)).toHaveLength(0); + }); + + it("should not include wake facility (commented out)", () => { + expect(API_ID_TO_NAME[384]).toBeUndefined(); + }); + + it("should not include facilities with API ID 0", () => { + expect(API_ID_TO_NAME[0]).toBeUndefined(); + }); + }); + + describe("FacilityMapper.getFacilityByName", () => { + it("should return correct facility config for buncombe", () => { + const buncombeConfig = FacilityMapper.getFacilityByName("buncombe"); + expect(buncombeConfig).toEqual({ + name: "buncombe", + apiId: 0, // Initially 0, loaded from Parameter Store + displayName: "Buncombe County", + }); + }); + + it("should be case insensitive", () => { + const buncombeConfig = FacilityMapper.getFacilityByName("BUNCOMBE"); + expect(buncombeConfig).toEqual({ + name: "buncombe", + apiId: 0, // Initially 0, loaded from Parameter Store + displayName: "Buncombe County", + }); + }); + + it("should return null for wake (commented out)", () => { + expect(FacilityMapper.getFacilityByName("wake")).toBeNull(); + }); + + it("should return null for invalid facility names", () => { + expect(FacilityMapper.getFacilityByName("invalid")).toBeNull(); + expect(FacilityMapper.getFacilityByName("")).toBeNull(); + }); + }); + + describe("FacilityMapper.getFacilityByApiId", () => { + it("should return null initially (no API IDs loaded)", () => { + const buncombeConfig = FacilityMapper.getFacilityByApiId(23); + expect(buncombeConfig).toBeNull(); + }); + + it("should return null for wake API ID (commented out)", () => { + expect(FacilityMapper.getFacilityByApiId(384)).toBeNull(); + }); + + it("should return null for invalid API IDs", () => { + expect(FacilityMapper.getFacilityByApiId(999)).toBeNull(); + expect(FacilityMapper.getFacilityByApiId(0)).toBeNull(); + }); + }); + + describe("FacilityMapper.getApiIdForFacility", () => { + it("should return 0 initially for buncombe (not loaded yet)", () => { + expect(FacilityMapper.getApiIdForFacility("buncombe")).toBe(0); + }); + + it("should return null for wake (commented out)", () => { + expect(FacilityMapper.getApiIdForFacility("wake")).toBeNull(); + }); + + it("should return null for invalid facility names", () => { + expect(FacilityMapper.getApiIdForFacility("invalid")).toBeNull(); + }); + }); + + describe("FacilityMapper.getNameForApiId", () => { + it("should return null initially (no API IDs loaded)", () => { + expect(FacilityMapper.getNameForApiId(23)).toBeNull(); + }); + + it("should return null for wake API ID (commented out)", () => { + expect(FacilityMapper.getNameForApiId(384)).toBeNull(); + }); + + it("should return null for invalid API IDs", () => { + expect(FacilityMapper.getNameForApiId(999)).toBeNull(); + expect(FacilityMapper.getNameForApiId(0)).toBeNull(); + }); + }); + + describe("FacilityMapper.isValidFacilityName", () => { + it("should return true for buncombe", () => { + expect(FacilityMapper.isValidFacilityName("buncombe")).toBe(true); + }); + + it("should be case insensitive", () => { + expect(FacilityMapper.isValidFacilityName("BUNCOMBE")).toBe(true); + expect(FacilityMapper.isValidFacilityName("Buncombe")).toBe(true); + }); + + it("should return false for wake (commented out)", () => { + expect(FacilityMapper.isValidFacilityName("wake")).toBe(false); + }); + + it("should return false for invalid facility names", () => { + expect(FacilityMapper.isValidFacilityName("invalid")).toBe(false); + expect(FacilityMapper.isValidFacilityName("")).toBe(false); + }); + }); + + describe("FacilityMapper.getAllFacilities", () => { + it("should return only buncombe facility", () => { + const facilities = FacilityMapper.getAllFacilities(); + expect(facilities).toHaveLength(1); + expect(facilities.map((f) => f.name)).toEqual(["buncombe"]); + }); + + it("should return facilities with correct structure", () => { + const facilities = FacilityMapper.getAllFacilities(); + facilities.forEach((facility) => { + expect(facility).toHaveProperty("name"); + expect(facility).toHaveProperty("apiId"); + expect(facility).toHaveProperty("displayName"); + expect(typeof facility.name).toBe("string"); + expect(typeof facility.apiId).toBe("number"); + expect(typeof facility.displayName).toBe("string"); + }); + }); + }); + + describe("FacilityMapper.getActiveFacilities", () => { + it("should return empty array initially (no API IDs loaded)", () => { + const activeFacilities = FacilityMapper.getActiveFacilities(); + expect(activeFacilities).toHaveLength(0); + }); + + it("should not include facilities with API ID 0", () => { + const activeFacilities = FacilityMapper.getActiveFacilities(); + const inactiveFacilities = activeFacilities.filter( + (f) => f.apiId === 0 + ); + expect(inactiveFacilities).toHaveLength(0); + }); + + it("should return facilities that can be used for data collection", () => { + const activeFacilities = FacilityMapper.getActiveFacilities(); + activeFacilities.forEach((facility) => { + expect(facility.apiId).toBeGreaterThan(0); + }); + }); + }); +}); diff --git a/serverless/lib/__tests__/StorageClient.test.ts b/serverless/lib/__tests__/StorageClient.test.ts new file mode 100644 index 0000000..2ec0716 --- /dev/null +++ b/serverless/lib/__tests__/StorageClient.test.ts @@ -0,0 +1,329 @@ +// Mock the AWS SDK BEFORE importing StorageClient +const mockDynamoDb = { + send: jest.fn(), +}; + +jest.mock("@aws-sdk/client-dynamodb", () => ({ + DynamoDBClient: jest.fn().mockImplementation(() => ({})), +})); + +jest.mock("@aws-sdk/lib-dynamodb", () => ({ + DynamoDBDocumentClient: { + from: jest.fn().mockReturnValue(mockDynamoDb), + }, + GetCommand: jest.fn().mockImplementation((params) => params), + PutCommand: jest.fn().mockImplementation((params) => params), + QueryCommand: jest.fn().mockImplementation((params) => params), + BatchWriteCommand: jest.fn().mockImplementation((params) => params), +})); + +// Now import after mocks are set up +import StorageClient from "../StorageClient"; +import { InmateRecord } from "../types"; + +describe("StorageClient", () => { + const mockInmate: InmateRecord = { + FirstName: "John", + LastName: "Doe", + MiddleName: "M", + TotalBondAmount: "1000", + ArrestDate: "9/15/2025 10:30:00 AM", + }; + + beforeEach(() => { + jest.clearAllMocks(); + // Set up environment variables + process.env.JAILDATA_TABLE = "test-table"; + process.env.AWS_REGION = "us-east-1"; + }); + + describe("saveInmate", () => { + it("should save an inmate record successfully", async () => { + mockDynamoDb.send.mockResolvedValueOnce({}); + + await StorageClient.saveInmate("wake", mockInmate); + + expect(mockDynamoDb.send).toHaveBeenCalledTimes(1); + }); + + it("should handle missing name fields", async () => { + const incompleteInmate: InmateRecord = { + TotalBondAmount: "500", + ArrestDate: "9/15/2025 10:30:00 AM", + }; + + mockDynamoDb.send.mockResolvedValueOnce({}); + + await StorageClient.saveInmate("wake", incompleteInmate); + + expect(mockDynamoDb.send).toHaveBeenCalledTimes(1); + }); + + it("should handle string and numeric bond amounts", async () => { + const inmateWithStringBond = { + ...mockInmate, + TotalBondAmount: "1500.50", + }; + const inmateWithNumericBond = { + ...mockInmate, + TotalBondAmount: 2000, + }; + + mockDynamoDb.send.mockResolvedValue({}); + + await StorageClient.saveInmate("wake", inmateWithStringBond); + await StorageClient.saveInmate("wake", inmateWithNumericBond); + + expect(mockDynamoDb.send).toHaveBeenCalledTimes(2); + }); + + it("should throw an error when DynamoDB operation fails", async () => { + const error = new Error("DynamoDB error"); + mockDynamoDb.send.mockRejectedValueOnce(error); + + await expect( + StorageClient.saveInmate("wake", mockInmate) + ).rejects.toThrow(); + }); + }); + + describe("getInmatesByFacility", () => { + it("should return inmates for a facility", async () => { + const mockItems = [ + { + PK: "INMATE#wake#DOE#JOHN#M", + SK: "2025-09-15", + totalBondAmount: 1000, + rawData: mockInmate, + }, + ]; + + mockDynamoDb.send.mockResolvedValueOnce({ Items: mockItems }); + + const result = await StorageClient.getInmatesByFacility("wake", 50); + + expect(result).toHaveLength(1); + expect(mockDynamoDb.send).toHaveBeenCalledTimes(1); + }); + + it("should handle empty results", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + const result = await StorageClient.getInmatesByFacility("wake"); + + expect(result).toHaveLength(0); + }); + + it("should use default limit of 100", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.getInmatesByFacility("wake"); + + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + Limit: 100, + }) + ); + }); + }); + + describe("getInmatesByFacilityAndDateRange", () => { + it("should query with start date only", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.getInmatesByFacilityAndDateRange( + "wake", + "2025-09-01" + ); + + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + KeyConditionExpression: + "GSI1PK = :gsi1pk AND GSI1SK >= :startDate", + ExpressionAttributeValues: { + ":gsi1pk": "FACILITY#wake", + ":startDate": "2025-09-01", + }, + }) + ); + }); + + it("should query with end date only", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.getInmatesByFacilityAndDateRange( + "wake", + undefined, + "2025-09-30" + ); + + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + KeyConditionExpression: + "GSI1PK = :gsi1pk AND GSI1SK <= :endDate", + ExpressionAttributeValues: { + ":gsi1pk": "FACILITY#wake", + ":endDate": "2025-09-30", + }, + }) + ); + }); + + it("should query with date range", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.getInmatesByFacilityAndDateRange( + "wake", + "2025-09-01", + "2025-09-30" + ); + + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + KeyConditionExpression: + "GSI1PK = :gsi1pk AND GSI1SK BETWEEN :startDate AND :endDate", + ExpressionAttributeValues: { + ":gsi1pk": "FACILITY#wake", + ":startDate": "2025-09-01", + ":endDate": "2025-09-30", + }, + }) + ); + }); + }); + + describe("batchSaveInmates", () => { + const mockInmates: InmateRecord[] = [ + { + FirstName: "John", + LastName: "Doe", + ArrestDate: "9/15/2025 10:30:00 AM", + }, + { + FirstName: "Jane", + LastName: "Smith", + ArrestDate: "9/16/2025 11:00:00 AM", + }, + ]; + + it("should batch save multiple inmates", async () => { + mockDynamoDb.send.mockResolvedValue({}); + + await StorageClient.batchSaveInmates("wake", mockInmates); + + expect(mockDynamoDb.send).toHaveBeenCalledTimes(1); + }); + + it("should chunk large batches into groups of 25", async () => { + // Create 30 inmates to test chunking + const largeInmateList = Array.from({ length: 30 }, (_, i) => ({ + FirstName: `Inmate${i}`, + LastName: "Test", + ArrestDate: "9/15/2025 10:30:00 AM", + })); + + mockDynamoDb.send.mockResolvedValue({}); + + await StorageClient.batchSaveInmates("wake", largeInmateList); + + // Should be called twice: 25 + 5 + expect(mockDynamoDb.send).toHaveBeenCalledTimes(2); + }); + + it("should handle empty inmate array", async () => { + await StorageClient.batchSaveInmates("wake", []); + + expect(mockDynamoDb.send).not.toHaveBeenCalled(); + }); + }); + + describe("searchInmatesByLastName", () => { + it("should search inmates by last name", async () => { + const mockItems = [ + { + PK: "INMATE#wake#SMITH#JOHN#", + SK: "2025-09-15", + rawData: { FirstName: "John", LastName: "Smith" }, + }, + ]; + + mockDynamoDb.send.mockResolvedValueOnce({ Items: mockItems }); + + const result = await StorageClient.searchInmatesByLastName( + "wake", + "Smith" + ); + + expect(result).toHaveLength(1); + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + KeyConditionExpression: "begins_with(PK, :pkPrefix)", + ExpressionAttributeValues: { + ":pkPrefix": "INMATE#wake#SMITH", + }, + }) + ); + }); + + it("should convert last name to uppercase for search", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.searchInmatesByLastName("wake", "smith"); + + expect(mockDynamoDb.send).toHaveBeenCalledWith( + expect.objectContaining({ + ExpressionAttributeValues: { + ":pkPrefix": "INMATE#wake#SMITH", + }, + }) + ); + }); + }); + + describe("getAllRecentInmates", () => { + it("should get recent inmates from last 24 hours", async () => { + mockDynamoDb.send.mockResolvedValueOnce({ Items: [] }); + + await StorageClient.getAllRecentInmates(50); + + const call = mockDynamoDb.send.mock.calls[0][0]; + expect(call.KeyConditionExpression).toContain( + "begins_with(GSI1PK, :facilityPrefix)" + ); + expect( + call.ExpressionAttributeValues[":facilityPrefix"] + ).toBe("FACILITY#"); + expect(call.Limit).toBe(50); + }); + }); + + describe("Date parsing functions", () => { + // Since parseArrestDate is not exported, we'll test it indirectly through saveInmate + it("should handle different date formats in saveInmate", async () => { + const inmateWithStandardDate = { + FirstName: "John", + LastName: "Doe", + ArrestDate: "9/15/2025 10:30:00 AM", + }; + + const inmateWithISODate = { + FirstName: "Jane", + LastName: "Doe", + ArrestDate: "2025-09-15T10:30:00Z", + }; + + const inmateWithNoDate = { + FirstName: "Bob", + LastName: "Doe", + }; + + mockDynamoDb.send.mockResolvedValue({}); + + await StorageClient.saveInmate("wake", inmateWithStandardDate); + await StorageClient.saveInmate("wake", inmateWithISODate); + await StorageClient.saveInmate("wake", inmateWithNoDate); + + expect(mockDynamoDb.send).toHaveBeenCalledTimes(3); + }); + }); +}); diff --git a/serverless/lib/types.ts b/serverless/lib/types.ts new file mode 100644 index 0000000..035faa8 --- /dev/null +++ b/serverless/lib/types.ts @@ -0,0 +1,79 @@ +// Common types for jail data processing + +// Response structure from jail API (used for both API responses and batch processing) +export interface InmateApiBatch { + Inmates: InmateRecord[]; + Total: number; + ShowImages: boolean; +} + +// Loose typing for inmate record as requested - keeping it flexible +export interface InmateRecord { + // We'll extract these as attributes for DynamoDB + FirstName?: string; + LastName?: string; + MiddleName?: string; + TotalBondAmount?: number | string; + ArrestDate?: string; // Format: M/d/yyyy h:mm:ss [AP]M + + // Allow any other properties that come from the API + [key: string]: unknown; +} + +// DynamoDB record structure (single table design) +export interface InmateDynamoRecord extends Record { + PK: string; // INMATE#{facilityName}#{lastName}#{firstName}#{middleName} + SK: string; // Record date in YYYY-MM-DD format (from ArrestDate, naturally sortable) + GSI1PK?: string; // FACILITY#{facilityName} (for facility-based queries) + GSI1SK?: string; // Record date in YYYY-MM-DD format (same as SK, for date-based sorting) + totalBondAmount?: number; + recordDate: string; // Record date in YYYY-MM-DD format (same as SK) + lastUpdated: string; // ISO timestamp of last update + rawData: InmateRecord; // Full original record +} + +// SQS message types +export interface BatchProcessingMessage { + facilityId: string; + batch: InmateApiBatch; + batchNumber: number; + totalBatches?: number; + requestId: string; // For tracking/correlation +} + +// Data collection configuration +export interface DataCollectionConfig { + baseUrl: string; + facilityId: string; + batchSize: number; // Default 100 from the API structure +} + +// Pagination options for the API request +export interface PagingOptions { + SortOptions: Array<{ + Name: string; + SortDirection: "Ascending" | "Descending"; + Sequence: number; + }>; + Take: number; + Skip: number; +} + +// Filter options for the API request +export interface FilterOptionsParameters { + IntersectionSearch: boolean; + SearchText: string; + Parameters: unknown[]; +} + +// Complete API request body structure +export interface InmateApiRequestBody { + FilterOptionsParameters: FilterOptionsParameters; + IncludeCount: boolean; + PagingOptions: PagingOptions; +} + +// Type aliases for backward compatibility and semantic clarity +export type InmateApiResponse = InmateApiBatch; // Alias for API responses + +// Note: Severity and AlertCategory are exported from AlertService.ts diff --git a/serverless/package-lock.json b/serverless/package-lock.json index e72bfd1..e12b485 100644 --- a/serverless/package-lock.json +++ b/serverless/package-lock.json @@ -8,15 +8,18 @@ "@aws-sdk/client-cloudwatch": "^3.782.0", "@aws-sdk/client-dynamodb": "^3.543.0", "@aws-sdk/client-sns": "^3.782.0", + "@aws-sdk/client-sqs": "^3.896.0", "@aws-sdk/client-ssm": "^3.782.0", "@aws-sdk/lib-dynamodb": "^3.543.0", - "axios": "^1.12.1" + "axios": "^1.12.1", + "uuid": "^13.0.0" }, "devDependencies": { "@eslint/js": "^9.24.0", "@types/aws-lambda": "^8.10.147", "@types/jest": "^29.5.14", "@types/node": "^22.13.5", + "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.29.0", "@typescript-eslint/parser": "^8.29.0", "eslint": "^9.24.0", @@ -833,6 +836,58 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-sqs": { + "version": "3.896.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.896.0.tgz", + "integrity": "sha512-8nDSfBkXcqK3uxZ5NxiYGeRtTHFbMM1thLHm/myDDQrgnRMyhMXSbe6lPmCosMRZdqjmT0+cB9fc/XZAxA71fg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.896.0", + "@aws-sdk/credential-provider-node": "3.896.0", + "@aws-sdk/middleware-host-header": "3.893.0", + "@aws-sdk/middleware-logger": "3.893.0", + "@aws-sdk/middleware-recursion-detection": "3.893.0", + "@aws-sdk/middleware-sdk-sqs": "3.896.0", + "@aws-sdk/middleware-user-agent": "3.896.0", + "@aws-sdk/region-config-resolver": "3.893.0", + "@aws-sdk/types": "3.893.0", + "@aws-sdk/util-endpoints": "3.895.0", + "@aws-sdk/util-user-agent-browser": "3.893.0", + "@aws-sdk/util-user-agent-node": "3.896.0", + "@smithy/config-resolver": "^4.2.2", + "@smithy/core": "^3.12.0", + "@smithy/fetch-http-handler": "^5.2.1", + "@smithy/hash-node": "^4.1.1", + "@smithy/invalid-dependency": "^4.1.1", + "@smithy/md5-js": "^4.1.1", + "@smithy/middleware-content-length": "^4.1.1", + "@smithy/middleware-endpoint": "^4.2.4", + "@smithy/middleware-retry": "^4.3.0", + "@smithy/middleware-serde": "^4.1.1", + "@smithy/middleware-stack": "^4.1.1", + "@smithy/node-config-provider": "^4.2.2", + "@smithy/node-http-handler": "^4.2.1", + "@smithy/protocol-http": "^5.2.1", + "@smithy/smithy-client": "^4.6.4", + "@smithy/types": "^4.5.0", + "@smithy/url-parser": "^4.1.1", + "@smithy/util-base64": "^4.1.0", + "@smithy/util-body-length-browser": "^4.1.0", + "@smithy/util-body-length-node": "^4.1.0", + "@smithy/util-defaults-mode-browser": "^4.1.4", + "@smithy/util-defaults-mode-node": "^4.1.4", + "@smithy/util-endpoints": "^3.1.2", + "@smithy/util-middleware": "^4.1.1", + "@smithy/util-retry": "^4.1.2", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-ssm": { "version": "3.896.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-ssm/-/client-ssm-3.896.0.tgz", @@ -1371,6 +1426,23 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-sqs": { + "version": "3.896.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.896.0.tgz", + "integrity": "sha512-Bb0UkqUK7EH6li4gxh3ANpcHZ0DMW8oIRbxColwoW+Hvxn7iVjNatZZTfcQu7shFhzzFcBmHljsXTSi4sS0BEA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "3.893.0", + "@smithy/smithy-client": "^4.6.4", + "@smithy/types": "^4.5.0", + "@smithy/util-hex-encoding": "^4.1.0", + "@smithy/util-utf8": "^4.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/middleware-ssec": { "version": "3.893.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.893.0.tgz", @@ -4099,7 +4171,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.1.1.tgz", "integrity": "sha512-MvWXKK743BuHjr/hnWuT6uStdKEaoqxHAQUvbKJPPZM5ZojTNFI5D+47BoQfBE5RgGlRRty05EbWA+NXDv+hIA==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.5.0", @@ -4769,6 +4840,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -10723,6 +10801,19 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz", + "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/serverless/package.json b/serverless/package.json index ef81b4b..60b0877 100644 --- a/serverless/package.json +++ b/serverless/package.json @@ -4,6 +4,7 @@ "@types/aws-lambda": "^8.10.147", "@types/jest": "^29.5.14", "@types/node": "^22.13.5", + "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^8.29.0", "@typescript-eslint/parser": "^8.29.0", "eslint": "^9.24.0", @@ -20,9 +21,11 @@ "@aws-sdk/client-cloudwatch": "^3.782.0", "@aws-sdk/client-dynamodb": "^3.543.0", "@aws-sdk/client-sns": "^3.782.0", + "@aws-sdk/client-sqs": "^3.896.0", "@aws-sdk/client-ssm": "^3.782.0", "@aws-sdk/lib-dynamodb": "^3.543.0", - "axios": "^1.12.1" + "axios": "^1.12.1", + "uuid": "^13.0.0" }, "scripts": { "test": "jest",