From a2e8e8923a7a3de6625e0e989368e511c419c8cb Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 14:52:49 -0400 Subject: [PATCH 01/23] create initial database docker compose --- docker-compose.yaml | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..ed795d7 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,98 @@ +services: + postgres: + image: postgres:15-alpine + restart: always + env_file: + - .env + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "$${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - dev_network + + opensearch-node1: + image: opensearchproject/opensearch:latest + container_name: opensearch-node1 + env_file: + - .env + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "DISABLE_SECURITY_PLUGIN=true" + - plugins.security.ssl.http.enabled=false + - plugins.security.ssl.transport.enabled=false + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"] + interval: 10s + timeout: 5s + retries: 5 + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data + expose: + - "9200" + - "9600" + networks: + - dev_network + + opensearch-node2: + image: opensearchproject/opensearch:latest + container_name: opensearch-node2 + env_file: + - .env + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "DISABLE_SECURITY_PLUGIN=true" + - plugins.security.ssl.http.enabled=false + - plugins.security.ssl.transport.enabled=false + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - opensearch-data2:/usr/share/opensearch/data + expose: + - "9200" + - "9600" + networks: + - dev_network + + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:latest + container_name: opensearch-dashboards + expose: + - "5601" + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' + networks: + - dev_network + +volumes: + postgres_data: + opensearch-data1: + opensearch-data2: + +networks: + dev_network: + name: dev_network + driver: bridge From 660f3ac484ae18dadf87fc0fd1acacff56391ae9 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 14:55:25 -0400 Subject: [PATCH 02/23] add cognito local --- cognito/config.json | 1 + cognito/db/clients.json | 30 ++++ cognito/db/local_2EfVJC8K.json | 270 +++++++++++++++++++++++++++++++++ docker-compose.yaml | 6 + 4 files changed, 307 insertions(+) create mode 100644 cognito/config.json create mode 100644 cognito/db/clients.json create mode 100644 cognito/db/local_2EfVJC8K.json diff --git a/cognito/config.json b/cognito/config.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/cognito/config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/cognito/db/clients.json b/cognito/db/clients.json new file mode 100644 index 0000000..9b91fc5 --- /dev/null +++ b/cognito/db/clients.json @@ -0,0 +1,30 @@ +{ + "Clients": { + "1r4k4b23bva9jj3kxgd28zcc3": { + "CallbackURLs": [ + "http://localhost:3000/callback" + ], + "ClientId": "1r4k4b23bva9jj3kxgd28zcc3", + "ClientName": "MyAppClient", + "ClientSecret": "5oolz6muthfeg2309apcivzel", + "CreationDate": "2025-04-15T18:12:43.442Z", + "ExplicitAuthFlows": [ + "ALLOW_USER_PASSWORD_AUTH", + "ALLOW_REFRESH_TOKEN_AUTH" + ], + "LastModifiedDate": "2025-04-15T18:12:43.442Z", + "LogoutURLs": [ + "http://localhost:3000/logout" + ], + "SupportedIdentityProviders": [ + "COGNITO" + ], + "TokenValidityUnits": { + "AccessToken": "hours", + "IdToken": "minutes", + "RefreshToken": "days" + }, + "UserPoolId": "local_2EfVJC8K" + } + } +} \ No newline at end of file diff --git a/cognito/db/local_2EfVJC8K.json b/cognito/db/local_2EfVJC8K.json new file mode 100644 index 0000000..2b096db --- /dev/null +++ b/cognito/db/local_2EfVJC8K.json @@ -0,0 +1,270 @@ +{ + "Users": { + "test": { + "Attributes": [ + { + "Name": "sub", + "Value": "d9d08aa3-889b-4e86-8a61-120da16d3f61" + } + ], + "Enabled": true, + "Password": "test", + "RefreshTokens": [ + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJjb2duaXRvOnVzZXJuYW1lIjoidGVzdCIsImlhdCI6MTc0NDc0MjQ5MCwianRpIjoiZWEyN2M0MjctMjQ3MC00YTMyLTgzOGQtMjk2NDlkNGIyMmQxIiwiZXhwIjoxNzQ1MzQ3MjkwLCJpc3MiOiJodHRwOi8vMC4wLjAuMDo5MjI5L2xvY2FsXzJFZlZKQzhLIn0.KyyKmoBvIynYYxjoVsCC1BObv8LW6MqQK5FxqXgz4Qe57nyTUhyE1Gx3Ho0LMkuacVECvss_zgIPyHgF9iF4fpslL-X3LCJnpuKf9awc60ASc6SCC3qR_4weprBGkmTdHYseSlFdCBB1ZWcye8K74g-v1L5AXs3tcIxMN0eqmFkW5cBxO7j5KDfHJkUCfFqWedMKesodzN5eCmiQclBIquuG2hmS32FzNtCBzgMe465M2gs-xBfh5rVDSk8D6Cmr9P4vEPIIlh8SREvAwF91epoOJs5QgLsgB463L-Y3rbVG50mfoKRmXP4ap7R3QsCFms9q7Rg1hf8RcMn6xy83LA", + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJjb2duaXRvOnVzZXJuYW1lIjoidGVzdCIsImlhdCI6MTc0NDc0MjUxMSwianRpIjoiNmFmMzg1ZTItYmVhZi00MzNhLWI3YjktNGFmZjg4OGQxZWExIiwiZXhwIjoxNzQ1MzQ3MzExLCJpc3MiOiJodHRwOi8vMC4wLjAuMDo5MjI5L2xvY2FsXzJFZlZKQzhLIn0.StWLlQN_aNPE3wmtPUH3RuU_ovnwCV0j-YMEIOgYUDgsy7rJReR2XlJ2eCb5VKGTJcwsBzRVfP8oLaOkZvgIeB5o4fnukpypidGJ6OKbyfji1UPSVsyDlCSI5fqwQUGyPFUvcdzfEwh670uhSZe1SxU386yxmbYCjD5OHgfnimpV5mJYOCydh4WDhhTOErIouJ_khwHlvmprQlsKOuKhm5P4wjXTKPdxAJ6vTvMRGj1Kxfz52e5YZW700U0P4Lw4kWO0FoTv-3D610h5FvSMAWpQ2p-NbMK7R9NYDiSWtEAyDmCAJ6ZH8CMIiOI4q5s3SvPKdxogP1sZ7buWaBhh6Q", + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJjb2duaXRvOnVzZXJuYW1lIjoidGVzdCIsImlhdCI6MTc0NDc0MjU3OCwianRpIjoiOTk0MTc5NzctNmM4Yy00ZjZlLWExNGYtYWI1NzhiMzUxZTRiIiwiZXhwIjoxNzQ1MzQ3Mzc4LCJpc3MiOiJodHRwOi8vMC4wLjAuMDo5MjI5L2xvY2FsXzJFZlZKQzhLIn0.nyiPCbOisV8md6KUY65Tii0-uY7wlY3Pvmb0HoFWgR9Yvv0oI87OTpRNt4_ywsTtn4lgdzZNnRSu-fo02G5rmrHr0y4CWpu_TMt6GOmsGBuJbWMd_DOElaDA70bhbYWW_HOfgw8W6O1CmcKq9LVr8I1XQa6FyhyTIoHYa3os-MrjpHpD7S0ARX5f9HFaMwFmhSZgCP2fIrkOBhrrWF-e4SYMTHQqsWUdJfXZMZa0a64CaK9-oUAUONKFZTppLnH2TeO39GROWdcjHl50v42mCNu_xgGIV2dquruWLXALs9gZkdU031QeK0-XrJoy_bDfh8lSNTR70uhHws0h5czlFg" + ], + "UserCreateDate": "2025-04-15T18:40:59.016Z", + "UserLastModifiedDate": "2025-04-15T18:40:59.016Z", + "Username": "test", + "UserStatus": "CONFIRMED", + "ConfirmationCode": "559558" + } + }, + "Options": { + "Policies": { + "PasswordPolicy": { + "MinimumLength": 8, + "RequireUppercase": true, + "RequireLowercase": true, + "RequireNumbers": true, + "RequireSymbols": true, + "TemporaryPasswordValidityDays": 7 + } + }, + "LambdaConfig": {}, + "SchemaAttributes": [ + { + "Name": "sub", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": false, + "Required": true, + "StringAttributeConstraints": { + "MinLength": "1", + "MaxLength": "2048" + } + }, + { + "Name": "name", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "given_name", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "family_name", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "middle_name", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "nickname", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "preferred_username", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "profile", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "picture", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "website", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "email", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "email_verified", + "AttributeDataType": "Boolean", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false + }, + { + "Name": "gender", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "birthdate", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "10", + "MaxLength": "10" + } + }, + { + "Name": "zoneinfo", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "locale", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "phone_number", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "phone_number_verified", + "AttributeDataType": "Boolean", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false + }, + { + "Name": "address", + "AttributeDataType": "String", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "StringAttributeConstraints": { + "MinLength": "0", + "MaxLength": "2048" + } + }, + { + "Name": "updated_at", + "AttributeDataType": "Number", + "DeveloperOnlyAttribute": false, + "Mutable": true, + "Required": false, + "NumberAttributeConstraints": { + "MinValue": "0" + } + } + ], + "VerificationMessageTemplate": { + "DefaultEmailOption": "CONFIRM_WITH_CODE" + }, + "MfaConfiguration": "OFF", + "EstimatedNumberOfUsers": 0, + "EmailConfiguration": { + "EmailSendingAccount": "COGNITO_DEFAULT" + }, + "AdminCreateUserConfig": { + "AllowAdminCreateUserOnly": false, + "UnusedAccountValidityDays": 7 + }, + "UsernameAttributes": [ + "username" + ], + "Arn": "arn:aws:cognito-idp:local:local:userpool/local_2EfVJC8K", + "CreationDate": "2025-04-15T18:09:47.423Z", + "Id": "local_2EfVJC8K", + "LastModifiedDate": "2025-04-15T18:09:47.423Z", + "Name": "MyUserPool" + } +} \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index ed795d7..3328765 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -86,6 +86,12 @@ services: OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' networks: - dev_network + cognito: + image: jagregory/cognito-local:latest + ports: + - "9229:9229" + volumes: + - ./cognito:/app/.cognito volumes: postgres_data: From c1940fb4a76e3de8a08e6cb3c03dad2c7f0e0087 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 15:03:20 -0400 Subject: [PATCH 03/23] create api docker file and docker compose --- api.Dockerfile | 12 ++++++++++++ docker-compose.yaml | 11 +++++++++++ 2 files changed, 23 insertions(+) create mode 100644 api.Dockerfile diff --git a/api.Dockerfile b/api.Dockerfile new file mode 100644 index 0000000..e522c0d --- /dev/null +++ b/api.Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.13-alpine + +RUN apk update && apk upgrade +RUN apk add --no-cache --virtual build-deps build-base gcc +RUN pip install aws-sam-cli +RUN apk del build-deps + +WORKDIR /app +COPY . . +RUN /usr/local/bin/sam build + +CMD ["/usr/local/bin/sam", "local", "start-api", "--host", "0.0.0.0", "--docker-network", "dev_network"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 3328765..cc58729 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -93,6 +93,17 @@ services: volumes: - ./cognito:/app/.cognito + api: + build: + dockerfile: ../api.Dockerfile + context: ./api + ports: + - "3000:3000" + networks: + - dev_network + volumes: + - /var/run/docker.sock:/var/run/docker.sock + volumes: postgres_data: opensearch-data1: From 323611e37dac03d4ab337ae70579b0387524f812 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 15:06:13 -0400 Subject: [PATCH 04/23] create website docker file and docker compose --- docker-compose.yaml | 13 +++++++++++++ website.Dockerfile | 14 ++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 website.Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index cc58729..3be31b9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -104,6 +104,19 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock + website: + build: + dockerfile: ../website.Dockerfile + context: ./website + env_file: + - .env + environment: + - VITE_GATEWAY_API_URL=api:3000 + ports: + - "5500:5500" + networks: + - dev_network + volumes: postgres_data: opensearch-data1: diff --git a/website.Dockerfile b/website.Dockerfile new file mode 100644 index 0000000..b84759a --- /dev/null +++ b/website.Dockerfile @@ -0,0 +1,14 @@ +FROM node:23-alpine + + +WORKDIR /app +COPY package.json . +RUN npm install + +COPY . . + +EXPOSE 5500 + +CMD ["/usr/local/bin/npm", "run", "docker"] + +# CMD ["npm ", "run", "dev"] From b1d48d443db256ba61d66aa88fca038a31dceff6 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 15:06:30 -0400 Subject: [PATCH 05/23] update website submodule --- website | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website b/website index 631f53f..16cb33c 160000 --- a/website +++ b/website @@ -1 +1 @@ -Subproject commit 631f53f56905f01232ac88d60ee04833aa59c307 +Subproject commit 16cb33ceb5275a4547b026682dee61e524e6c19e From b7ab7131503f81f155940ed78e6200539dfcf95f Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 15:15:57 -0400 Subject: [PATCH 06/23] recursive submodule update --- Submodules.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Submodules.sh b/Submodules.sh index 57da730..5b5110f 100755 --- a/Submodules.sh +++ b/Submodules.sh @@ -1,4 +1 @@ -git submodule update --init - -git submodule foreach git submodule update --init - +git submodule update --init --recursive From 4a1033f1be1057ea4b26aa9b4b318597436b2439 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 17:49:29 -0400 Subject: [PATCH 07/23] remove api running sam local in a docker container is really jank and can lead to strange issues, i tried for about an hour to get it to work --- docker-compose.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3be31b9..0943226 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -92,17 +92,7 @@ services: - "9229:9229" volumes: - ./cognito:/app/.cognito - - api: - build: - dockerfile: ../api.Dockerfile - context: ./api - ports: - - "3000:3000" - networks: - - dev_network - volumes: - - /var/run/docker.sock:/var/run/docker.sock + website: build: From a6ae349c2387d9a186ef7108e6414a05b1b9d9f0 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 15 Apr 2025 17:49:46 -0400 Subject: [PATCH 08/23] remove env variable --- docker-compose.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 0943226..56bc7ba 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -100,8 +100,6 @@ services: context: ./website env_file: - .env - environment: - - VITE_GATEWAY_API_URL=api:3000 ports: - "5500:5500" networks: From a0babf22e5c280f06fd1baf27ada4fca45a60349 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 16 Apr 2025 12:23:01 -0400 Subject: [PATCH 09/23] open postgres port --- docker-compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 56bc7ba..eff1de2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,6 +2,8 @@ services: postgres: image: postgres:15-alpine restart: always + ports: + - "5432:5432" env_file: - .env volumes: @@ -92,7 +94,6 @@ services: - "9229:9229" volumes: - ./cognito:/app/.cognito - website: build: From c76aa9e85caa24b8e44f341250b01e3880d48bec Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 16 Apr 2025 13:51:20 -0400 Subject: [PATCH 10/23] remove api dockerfile --- api.Dockerfile | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 api.Dockerfile diff --git a/api.Dockerfile b/api.Dockerfile deleted file mode 100644 index e522c0d..0000000 --- a/api.Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.13-alpine - -RUN apk update && apk upgrade -RUN apk add --no-cache --virtual build-deps build-base gcc -RUN pip install aws-sam-cli -RUN apk del build-deps - -WORKDIR /app -COPY . . -RUN /usr/local/bin/sam build - -CMD ["/usr/local/bin/sam", "local", "start-api", "--host", "0.0.0.0", "--docker-network", "dev_network"] \ No newline at end of file From 4d0dc5caf77f66ea7c61a9bc74e11ae101981b1f Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 16 Apr 2025 13:51:39 -0400 Subject: [PATCH 11/23] update website submodule --- website | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website b/website index 16cb33c..6fc4be0 160000 --- a/website +++ b/website @@ -1 +1 @@ -Subproject commit 16cb33ceb5275a4547b026682dee61e524e6c19e +Subproject commit 6fc4be0b1e4ad2b8cab9a35531770f45d1a71d93 From 8e6e6719198f853f13af53925dece236f557f3f3 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Fri, 18 Apr 2025 12:03:59 -0400 Subject: [PATCH 12/23] change dashes to underscores --- .gitmodules | 2 +- data-product-kit => data_product_kit | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename data-product-kit => data_product_kit (100%) diff --git a/.gitmodules b/.gitmodules index bb93b6c..0ea0406 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ url = git@github.com:mirrulations/transformation_trigger.git branch = main [submodule "data-product-kit"] - path = data-product-kit + path = data_product_kit url = git@github.com:mirrulations/Data-Product-Kit.git branch = main [submodule "website"] diff --git a/data-product-kit b/data_product_kit similarity index 100% rename from data-product-kit rename to data_product_kit From 93949525719fc249ca1ec8068a405da73dd48938 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:21:33 -0400 Subject: [PATCH 13/23] general cleanup --- .gitignore | 4 +++- Submodules.sh | 1 - docker-compose.yaml | 15 ++++----------- 3 files changed, 7 insertions(+), 13 deletions(-) delete mode 100755 Submodules.sh diff --git a/.gitignore b/.gitignore index a7dfb30..14cfef6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .DS_Store -.env \ No newline at end of file +.env +.env.json +.aws-sam diff --git a/Submodules.sh b/Submodules.sh deleted file mode 100755 index 5b5110f..0000000 --- a/Submodules.sh +++ /dev/null @@ -1 +0,0 @@ -git submodule update --init --recursive diff --git a/docker-compose.yaml b/docker-compose.yaml index eff1de2..2e7ea3f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,6 @@ services: postgres: image: postgres:15-alpine - restart: always ports: - "5432:5432" env_file: @@ -9,7 +8,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD", "pg_isready", "-U", "$${POSTGRES_USER}"] + test: ["CMD", "pg_isready", "-U", "$POSTGRES_USER"] interval: 10s timeout: 5s retries: 5 @@ -19,6 +18,9 @@ services: opensearch-node1: image: opensearchproject/opensearch:latest container_name: opensearch-node1 + ports: + - "9200:9200" + - "9600:9600" env_file: - .env environment: @@ -79,15 +81,6 @@ services: networks: - dev_network - opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:latest - container_name: opensearch-dashboards - expose: - - "5601" - environment: - OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' - networks: - - dev_network cognito: image: jagregory/cognito-local:latest ports: From 04926a4a748b87c8e0d38ab23631401ede057293 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:21:44 -0400 Subject: [PATCH 14/23] create env json creation script --- createEnvJson.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 createEnvJson.py diff --git a/createEnvJson.py b/createEnvJson.py new file mode 100644 index 0000000..8a88f6f --- /dev/null +++ b/createEnvJson.py @@ -0,0 +1,40 @@ +import json + +LAMBDA_FUNCTIONS = [ + "ApiFunction", + "OrchestratorFunction", + "SQLDocketIngestFunction", + "SQLDocumentIngestFunction", + "SQLCommentIngestFunction", + "HTMSummaryIngestFunction", + "OpenSearchCommentFunction", + "OpenSearchTextExtractFunction", +] + + +def main(): + env_json = {function: {} for function in LAMBDA_FUNCTIONS} + try: + with open(".env", "r") as env: + print("creating .env.json from .env file") + for line in env: + line = line.strip() + if line.startswith("#"): + # skip commented out lines + continue + line_components = line.split("=") + key = line_components[0] + value = "".join(line_components[1:]) + if line: + for function in LAMBDA_FUNCTIONS: + env_json[function][key] = value + except FileNotFoundError: + print("could not find .env file") + exit(1) + with open(".env.json", "w") as f: + f.write(json.dumps(env_json, indent=2)) + print("created .env.json") + + +if __name__ == "__main__": + main() From c27d416c7e900f44b84b97dd043438119977a7c2 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:21:56 -0400 Subject: [PATCH 15/23] create ingest script --- ingest.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 100 insertions(+) create mode 100755 ingest.py create mode 100644 requirements.txt diff --git a/ingest.py b/ingest.py new file mode 100755 index 0000000..17233b6 --- /dev/null +++ b/ingest.py @@ -0,0 +1,99 @@ +import sys +import json +import boto3 +import pathlib + + +def generate_payload(path) -> str: + payload = { + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": {"principalId": "EXAMPLE"}, + "requestParameters": {"sourceIPAddress": "127.0.0.1"}, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH", + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "mirrulations", + "ownerIdentity": {"principalId": "EXAMPLE"}, + "arn": "arn:aws:s3:::mirrulations", + }, + "object": { + "key": f"{path}", + "size": 1024, + "eTag": "0123456789abcdef0123456789abcdef", + "sequencer": "0A1B2C3D4E5F678901", + }, + }, + } + ] + } + return payload + + +def get_s3_path_from_filepath(file: pathlib.Path) -> str: + filepath = file.name + if filepath.endswith(".htm"): + #ACF-2025-0004-0001_content.htm + split_file = filepath.split("-") + agency = split_file[0] + docket_id = "-".join( + split_file[:-1] + ) # join everything but the last element in the list + type = "documents" + + elif filepath.endswith(".json"): + with open(file, "r") as f: + data = json.load(f) + type = data["data"]["type"] + attributes = data["data"]["attributes"] + + agency = attributes["agencyId"] + + if type == "dockets": + type = "docket" + docket_id = data["data"]["id"] + else: + docket_id = attributes["docketId"] + else: + return None + return f"raw-data/{agency}/{docket_id}/text-{docket_id}/{type}/{filepath}" + + +def invoke_local_lambda(payload: str) -> None: + lambda_client = boto3.client("lambda", endpoint_url="http://localhost:3001") + lambda_client.invoke( + FunctionName="OrchestratorFunction", + Payload=bytes(json.dumps(payload), encoding="utf-8"), + ) + + +def main(): + """ + Main function to execute the file reading. Prompts the user for a + filepath, reads the file, and prints the contents. + """ + if len(sys.argv) > 1: + filepath = sys.argv[1] # Get filepath from command line argument + else: + print("Please enter a filepath") + + filepath = pathlib.Path(filepath) + + s3_path = get_s3_path_from_filepath(filepath) + payload = generate_payload(s3_path) + + invoke_local_lambda(payload) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1db657b --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +boto3 \ No newline at end of file From 2ba176666d9114f5ae26b34ff40e6cec2d660c93 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:22:09 -0400 Subject: [PATCH 16/23] remove comment --- website.Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/website.Dockerfile b/website.Dockerfile index b84759a..9509d37 100644 --- a/website.Dockerfile +++ b/website.Dockerfile @@ -9,6 +9,4 @@ COPY . . EXPOSE 5500 -CMD ["/usr/local/bin/npm", "run", "docker"] - -# CMD ["npm ", "run", "dev"] +CMD ["/usr/local/bin/npm", "run", "docker"] \ No newline at end of file From 03a35c73832828b7f90a2014693be29e5031ddbe Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:22:30 -0400 Subject: [PATCH 17/23] update README with setup instructions --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fbef760..2aa6dfa 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,96 @@ # dev +## Setup + Requirements -- Docker -- Docker Compose +- Docker & Docker Compose +- AWS SAM -To get started clone this repository and run the following commands: +To get started clone this repository and run the following command: ```bash -chmod u+x Submodules.sh -./Submodules.sh +git submodule update --init --recursive +``` +this will initalize all the submodules in this repo. + +### Environment variables: + +Create a .env file with the following values, values with <> surrounding them will require you to fill them in. +``` +VITE_GATEWAY_API_URL=http://localhost:3000/dummy +VITE_COGNITO_USER_POOL_ENDPOINT=http://localhost:9229 +VITE_COGNITO_USER_POOL_ID=local_2EfVJC8K +VITE_COGNITO_CLIENT_ID=1r4k4b23bva9jj3kxgd28zcc3 +VITE_LOCAL=true + +ENVIRONMENT=local +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD= +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +OPENSEARCH_INITIAL_ADMIN_PASSWORD= +OPENSEARCH_HOST=opensearch-node1 +OPENSEARCH_PORT=9200 + +AWS_DEFAULT_REGION=us-east-1 ``` -This will clone the submodules +then run the `createEnvJson.py` file to create a .env.json file for the lambdas to use. + + +### Initializing the Databases +**THIS ONLY NEEDS TO BE DONE ON THE INITIAL SETUP** + +- start the databases with `docker compose up -d --build postgres opensearch-node1 opensearch-node2` + +#### Postgres +1. cd into `data_product_kit/sql` +2. run `POSTGRES_HOST=localhost python3 ResetDatabase.py` + - note: the `POSTGRES_HOST=localhost` is required + +#### Opensearch +- There is no initalization step because indexes are created when first ingesting data. + + +### Starting the frontend + +- start the frontend with `docker compose up -d --build website cognito` + +- open your browser of choice and navigate to `localhost:5500` + +- login with username `test` and password `test` + + +### Starting the API Gateway +- NOTE: This will require another terminal, as it does not run detached + +1. cd into `api` +2. run `sam build` +3. run `sam local start-api --docker-network dev_network --env-vars ../.env.json` + - this starts the api in the docker network that the DBs are in and with the environment variables in .env.json + + +### Starting the Orchestrator Lambda +- NOTE: This will rquire another terminal, as it does not run detached + +1. cd into `transformation_trigger/dev-env` +2. run `sam build` +3. run `sam local start-lambda --docker-network dev_network --container-env-vars ../../.env.json --env-vars ../../.env.json` + + + +### Ingesting Data +- The `ingest.py` file will invoke the orchestrator lambda with a given file. +- this requires the databases and the orchestrator lambda to be running +- this requires the `boto3` library + 1. create a virtual environment with `python3 -m venv .venv` + 2. activate the virtual environment with `source .venv/bin/activate` + 3. install the requirements file with `pip install -r requirements.txt` +- to ingest run `python3 ingest.py ` +- **NOTE**: When ingesting dockets, documents and comments, dockets need to be ingested before documents and documents need to be ingested before comments, due to the relational database structure. From 0cca39937c20321164eb1443b86b8122de2daef2 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Tue, 22 Apr 2025 18:43:40 -0400 Subject: [PATCH 18/23] add python3 as a requirement --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2aa6dfa..a1d0441 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Requirements - Docker & Docker Compose - AWS SAM +- python3 To get started clone this repository and run the following command: From caf536a6ccda3baf231a99a15fe6fcc78d97a28c Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 23 Apr 2025 10:19:56 -0400 Subject: [PATCH 19/23] update submodules --- api | 2 +- data_product_kit | 2 +- transformation_trigger | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api b/api index 5a6c382..928ea1b 160000 --- a/api +++ b/api @@ -1 +1 @@ -Subproject commit 5a6c38283010980af7aa8a1e4dd29e089139b03c +Subproject commit 928ea1bf4c422f9f415c1ff63d7b501f9476aa0d diff --git a/data_product_kit b/data_product_kit index 399e276..9e3ddc3 160000 --- a/data_product_kit +++ b/data_product_kit @@ -1 +1 @@ -Subproject commit 399e276a4d9f2aa1f279dd15810e9a7c5c81a96b +Subproject commit 9e3ddc35252f24122134785e763e86fcf5cc5979 diff --git a/transformation_trigger b/transformation_trigger index aec507a..0c962bc 160000 --- a/transformation_trigger +++ b/transformation_trigger @@ -1 +1 @@ -Subproject commit aec507a0bfa06f41d0bdec232caa85e8f68d6ce8 +Subproject commit 0c962bcc8ed0cdbc49f565c14079ad9fbf6abea6 From ad9ed5dac60bd5b6939694c7dc1ad3185e030863 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 23 Apr 2025 11:08:29 -0400 Subject: [PATCH 20/23] update submodules to use https instead of ssh --- .gitmodules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0ea0406..afa54b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,16 @@ [submodule "api"] path = api - url = git@github.com:mirrulations/API.git + url = https://github.com/mirrulations/API.git branch = main [submodule "transformation_trigger"] path = transformation_trigger - url = git@github.com:mirrulations/transformation_trigger.git + url = https://github.com/mirrulations/transformation_trigger.git branch = main [submodule "data-product-kit"] path = data_product_kit - url = git@github.com:mirrulations/Data-Product-Kit.git + url = https://github.com/mirrulations/Data-Product-Kit.git branch = main [submodule "website"] path = website - url = git@github.com:mirrulations/mirrulations-website.git + url = https://github.com/mirrulations/mirrulations-website.git branch = main From 71d4faa8ec7b57ded66c127ae732fd26fc9b856e Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 23 Apr 2025 11:21:30 -0400 Subject: [PATCH 21/23] add creating virtual environment step to postgres --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a1d0441..0ef07a4 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,13 @@ then run the `createEnvJson.py` file to create a .env.json file for the lambdas - start the databases with `docker compose up -d --build postgres opensearch-node1 opensearch-node2` #### Postgres -1. cd into `data_product_kit/sql` -2. run `POSTGRES_HOST=localhost python3 ResetDatabase.py` +1. cd into `data_product_kit` +2. setup a python virtual environment + 1. create a virtual environment with `python3 -m venv .venv` + 2. activate the virtual environment with `source .venv/bin/activate` + 3. install the requirements file with `pip install -r requirements.txt` +3. cd into `sql` +4. run `POSTGRES_HOST=localhost python3 ResetDatabase.py` - note: the `POSTGRES_HOST=localhost` is required #### Opensearch From 23581b23e1501b8abcb60d2fd5864b3d85938c0d Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 23 Apr 2025 11:35:19 -0400 Subject: [PATCH 22/23] add directory clarification --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0ef07a4..bd1bffe 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ then run the `createEnvJson.py` file to create a .env.json file for the lambdas ### Starting the frontend +- **NOTE**: Make sure you are not in any subfolders of the project for running this. + - if you run into an issue where docker cannot find a .env, you are not in the right spot. - start the frontend with `docker compose up -d --build website cognito` - open your browser of choice and navigate to `localhost:5500` From da25e66efac7cc6e245ffc67b6a2fa5548150e77 Mon Sep 17 00:00:00 2001 From: Owen Halliday Date: Wed, 23 Apr 2025 11:41:07 -0400 Subject: [PATCH 23/23] add root clarification --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bd1bffe..e5abd63 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ AWS_DEFAULT_REGION=us-east-1 then run the `createEnvJson.py` file to create a .env.json file for the lambdas to use. +### **EACH OF THE FOLLOWING STEPS ASSUMES THAT YOU ARE AT THE ROOT/TOP LEVEL OF THE PROJECT** + ### Initializing the Databases **THIS ONLY NEEDS TO BE DONE ON THE INITIAL SETUP**