From 9b09a061062291c20c0c504850dbc605d0aa53d3 Mon Sep 17 00:00:00 2001 From: Ajinkya Ghonge Date: Mon, 18 Sep 2023 09:35:52 -0700 Subject: [PATCH 1/2] Add ALS repository to KIA deployment script. Differential Revision: D49209824 fbshipit-source-id: a64b2eda2df723b205a652860b93496df586c052 --- fbpcs/infra/cloud_bridge/Dockerfile | 3 +++ fbpcs/infra/cloud_bridge/Makefile | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/fbpcs/infra/cloud_bridge/Dockerfile b/fbpcs/infra/cloud_bridge/Dockerfile index 70027b2ca..e21d84d4b 100644 --- a/fbpcs/infra/cloud_bridge/Dockerfile +++ b/fbpcs/infra/cloud_bridge/Dockerfile @@ -82,6 +82,9 @@ RUN pip3 install \ --only-binary=:all: --upgrade \ --target awsbundle \ cryptography -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ + +RUN pip install pyqldb -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ +RUN pip3 install pyion2json -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ # ######################################### # Spring Boot # ######################################### diff --git a/fbpcs/infra/cloud_bridge/Makefile b/fbpcs/infra/cloud_bridge/Makefile index 023103003..67de852df 100644 --- a/fbpcs/infra/cloud_bridge/Makefile +++ b/fbpcs/infra/cloud_bridge/Makefile @@ -68,11 +68,21 @@ kia_source_code: mkdir -p key_injection_agent/kia_source_code mkdir -p key_injection_agent/kia_source_code/private_computation mkdir -p key_injection_agent/kia_source_code/private_computation/tee_lift + mkdir -p key_injection_agent/kia_source_code/smart/ + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/ + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/audit_log_service + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs/repository + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs/entity + mkdir -p key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs/entity/measurement chmod +x key_injection_agent/kia_source_code cp -r ../../../private_computation/tee_lift/key_injection_agent/kia_runner.py key_injection_agent/kia_source_code/ cp -r ../../../private_computation/tee_lift/key_injection_agent key_injection_agent/kia_source_code/private_computation/tee_lift cp -r ../../../private_computation/tee_lift/pc_crypto key_injection_agent/kia_source_code/private_computation/tee_lift cp -r ../../../private_computation/tee_lift/utils key_injection_agent/kia_source_code/private_computation/tee_lift + cp -r ../../../smart/private_computation/audit_log_service/srcs/repository key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs + cp -r ../../../smart/private_computation/audit_log_service/srcs/entity key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs + clean_up_agent_source_code: mkdir -p clean_up_agent/clean_up_agent_source_code From fb55a4bd8ea92d9b14a516a52c750744b9989ec3 Mon Sep 17 00:00:00 2001 From: Ajinkya Ghonge Date: Mon, 18 Sep 2023 09:36:20 -0700 Subject: [PATCH 2/2] Deployment changes for MVA lambda function. Summary: # Context As part of KIA - ALS integration. I added logic required for PCR measurement validation to KIA lambda function. As part of this integration, the Lambda function needs to temporarily assume a different IAM role to query the Meta AWS deployed QLDB, validate the measurements and then assume it's original IAM role. While the KIA-ALS integration worked fine, the function was not able to assume it's original role post that. Based on further investigation I see hopping back and forth between IAM roles is not supported right now in AWS Lambda function. In order to solve this issue, I am moving the measurement validation logic to a new lambda that will be invoked from KIA. Thus, KIA will now invoke this new lambda with the PCRs and QLDB parameters. The new Measurement Validation Agent, will assume the role provided, validate the measurements and return back Success/Failure status back to KIA. Based on the Successful measurement validation then, KIA will proceed with the encryption and on Faillure will terminate the process with a Failed status. # Changes in the stack 1. Add a new QLDB repo handler : This handler will create the ALS QLDB repository. 2. Add a Measurement validation handler : This handler will hold logic to validate the measurements. 3. Add Measurement validation Runner : Entry point of the lambda function, this will validate the input and call the handlers. 4. Add Deployment changes for the new lambda : Changes need to deploy the new lambda as part of CB AWS infra. 5. Invoke Lambda from KIA : Add changes required to invoke the MVA lambda from KIA. 6. Add deployment changes for KIA : As part of this, we will need to pass the MVA lambda function name to KIA, add changes to deployment script for that. 7. Remove Measurement validation code from KIA : Now that the measurement validation logic is moved to a new lambda function, remove it from KIA. 8. Changes to undeploy MVA lambda : Add changes to undeploy MVA lambda when CB is uninstalled. 9. Add changes to CB API to pass QLDB parameters to KIA. 10. Add changes to Coordinator to pass QLDB parameters to CB. # Changes in this diff Add Deployment changes for the new lambda : Changes need to deploy the new lambda as part of CB AWS infra. Differential Revision: D49374679 fbshipit-source-id: 75f4b1af783351f9d316b5d470dcd921bf5f7d3e --- fbpcs/infra/cloud_bridge/Dockerfile | 8 +- fbpcs/infra/cloud_bridge/Makefile | 22 +++- fbpcs/infra/cloud_bridge/deploy_pc_infra.sh | 25 +++++ .../measurement_validation_agent/main.tf | 102 ++++++++++++++++++ .../measurement_validation_agent/output.tf | 4 + .../measurement_validation_agent/variable.tf | 34 ++++++ 6 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 fbpcs/infra/cloud_bridge/measurement_validation_agent/main.tf create mode 100644 fbpcs/infra/cloud_bridge/measurement_validation_agent/output.tf create mode 100644 fbpcs/infra/cloud_bridge/measurement_validation_agent/variable.tf diff --git a/fbpcs/infra/cloud_bridge/Dockerfile b/fbpcs/infra/cloud_bridge/Dockerfile index e21d84d4b..e9d9935c4 100644 --- a/fbpcs/infra/cloud_bridge/Dockerfile +++ b/fbpcs/infra/cloud_bridge/Dockerfile @@ -64,6 +64,7 @@ COPY aws_terraform_template /terraform_deployment/terraform_scripts COPY data_ingestion /terraform_deployment/terraform_scripts/data_ingestion COPY key_injection_agent /terraform_deployment/terraform_scripts/key_injection_agent COPY clean_up_agent /terraform_deployment/terraform_scripts/clean_up_agent +COPY measurement_validation_agent /terraform_deployment/terraform_scripts/measurement_validation_agent COPY semi_automated_data_ingestion /terraform_deployment/terraform_scripts/semi_automated_data_ingestion COPY config.yml /terraform_deployment/config COPY cli.py /terraform_deployment @@ -83,8 +84,11 @@ RUN pip3 install \ --target awsbundle \ cryptography -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ -RUN pip install pyqldb -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ -RUN pip3 install pyion2json -t /terraform_deployment/terraform_scripts/key_injection_agent/kia_source_code/ +RUN pip install pyqldb -t /terraform_deployment/terraform_scripts/measurement_validation_agent/mva_source_code/ +RUN pip3 install pyion2json -t /terraform_deployment/terraform_scripts/measurement_validation_agent/mva_source_code/ +RUN pip3 install dataclasses-json -t /terraform_deployment/terraform_scripts/measurement_validation_agent/mva_source_code/ +RUN pip3 install injector -t /terraform_deployment/terraform_scripts/measurement_validation_agent/mva_source_code/ + # ######################################### # Spring Boot # ######################################### diff --git a/fbpcs/infra/cloud_bridge/Makefile b/fbpcs/infra/cloud_bridge/Makefile index 67de852df..624f92c24 100644 --- a/fbpcs/infra/cloud_bridge/Makefile +++ b/fbpcs/infra/cloud_bridge/Makefile @@ -41,6 +41,7 @@ image-build: $(SERVER_JAR) external_deps @echo "\nCleaning up dependencies..." $(RM) -r aws_terraform_template $(RM) -r key_injection_agent/kia_source_code + $(RM) -r measurement_validation_agent/mva_source_code $(RM) -r clean_up_agent/clean_up_agent_source_code $(RM) config.yml @echo "Done" @@ -61,7 +62,7 @@ distclean: clean # Dockerfile will not accept these resources as links, so they need to be copied in -external_deps: kia_source_code clean_up_agent_source_code config.yml aws_terraform_template +external_deps: mva_source_code kia_source_code clean_up_agent_source_code config.yml aws_terraform_template @echo "Dependencies Copied\n" kia_source_code: @@ -84,6 +85,25 @@ kia_source_code: cp -r ../../../smart/private_computation/audit_log_service/srcs/entity key_injection_agent/kia_source_code/smart/private_computation/audit_log_service/srcs +mva_source_code: + mkdir -p measurement_validation_agent/mva_source_code + mkdir -p measurement_validation_agent/mva_source_code/private_computation + mkdir -p measurement_validation_agent/mva_source_code/private_computation/tee_lift + mkdir -p measurement_validation_agent/mva_source_code/smart/ + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/ + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs/repository + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs/entity + mkdir -p measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs/entity/measurement + chmod +x measurement_validation_agent/mva_source_code + cp -r ../../../private_computation/tee_lift/measurement_validation_agent/measurement_validation_runner.py measurement_validation_agent/mva_source_code/ + cp -r ../../../private_computation/tee_lift/measurement_validation_agent measurement_validation_agent/mva_source_code/private_computation/tee_lift + cp -r ../../../private_computation/tee_lift/pc_crypto measurement_validation_agent/mva_source_code/private_computation/tee_lift + cp -r ../../../private_computation/tee_lift/utils measurement_validation_agent/mva_source_code/private_computation/tee_lift + cp -r ../../../smart/private_computation/audit_log_service/srcs/repository measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs + cp -r ../../../smart/private_computation/audit_log_service/srcs/entity measurement_validation_agent/mva_source_code/smart/private_computation/audit_log_service/srcs + clean_up_agent_source_code: mkdir -p clean_up_agent/clean_up_agent_source_code mkdir -p clean_up_agent/clean_up_agent_source_code/private_computation diff --git a/fbpcs/infra/cloud_bridge/deploy_pc_infra.sh b/fbpcs/infra/cloud_bridge/deploy_pc_infra.sh index 5b8be186c..696f59604 100755 --- a/fbpcs/infra/cloud_bridge/deploy_pc_infra.sh +++ b/fbpcs/infra/cloud_bridge/deploy_pc_infra.sh @@ -394,6 +394,30 @@ deploy_aws_resources() { semi_automated_glue_job_arn=$(terraform output semi_automated_glue_job_arn | tr -d '"') fi + echo "######################## Deploying Measurment verification Agent Agent AWS Lambda" + cd /terraform_deployment/terraform_scripts/measurement_validation_agent + + log_streaming_data "starting to deploy Measurement verification agent." + + terraform init -reconfigure \ + -backend-config "bucket=$s3_bucket_config" \ + -backend-config "region=$region" \ + -backend-config "key=tfstate/measurement_verification_agent_$tag_postfix.tfstate" + + terraform apply \ + -auto-approve \ + -var "region=$region" \ + -var "tag_postfix=$tag_postfix" \ + -var "aws_account_id=$aws_account_id" \ + -var "measurement_validation_agent_lambda_function_name=$measurement_validation_agent_lambda_function_name" \ + -var "measurement_validation_agent_lambda_input_bucket=$s3_bucket_data" \ + -var "measurement_validation_agent_lambda_source_bucket=$s3_bucket_config" \ + -var "measurement_validation_agent_lambda_s3_key=mva_source.zip" + + log_streaming_data "deployed measurement verification agent." + + echo "######################## Deployed Measurement Verification Agent AWS Lambda" + echo "######################## Deploying Clean Up Agent Agent AWS Lambda" cd /terraform_deployment/terraform_scripts/clean_up_agent @@ -584,6 +608,7 @@ query_results_key_path="query-results" data_ingestion_lambda_name="cb-data-ingestion-stream-processor${tag_postfix}" kia_lambda_function_name="cb-kia${tag_postfix}" clean_up_agent_lambda_function_name="cb-clean-up-agent${tag_postfix}" +measurement_validation_agent_lambda_function_name="measurement_validation_agent${tag_postfix}" fb_pc_iam_policy="/terraform_deployment/fbpcs/infra/cloud_bridge/deployment_helper/aws/iam_policies/fb_pc_iam_policy_no_compute.json" fb_pc_data_bucket_policy="/terraform_deployment/fbpcs/infra/cloud_bridge/deployment_helper/aws/iam_policies/fb_pc_data_bucket_policy.json" data_bucket_policy_name="fb-pc-data-bucket-policy${tag_postfix}" diff --git a/fbpcs/infra/cloud_bridge/measurement_validation_agent/main.tf b/fbpcs/infra/cloud_bridge/measurement_validation_agent/main.tf new file mode 100644 index 000000000..b2a99b178 --- /dev/null +++ b/fbpcs/infra/cloud_bridge/measurement_validation_agent/main.tf @@ -0,0 +1,102 @@ +provider "aws" { + profile = "default" + region = var.region +} + +provider "archive" {} + +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +data "archive_file" "zip_lambda" { + type = "zip" + source_dir = "mva_source_code" + output_path = "mva_source.zip" +} + +resource "aws_s3_bucket_object" "upload_lambda" { + bucket = var.measurement_validation_agent_lambda_source_bucket + key = var.measurement_validation_agent_lambda_s3_key + source = "mva_source.zip" +} + +locals { + measurement_validation_agent_lambda_log_group = "/aws/lambda/${var.measurement_validation_agent_lambda_function_name}" + measurement_validation_agent_lambda_stream_name = "measurement-validation-agent-lambda-log-stream" +} + +resource "aws_cloudwatch_log_group" "measurement-validation-agent-lambda-log-group" { + name = local.measurement_validation_agent_lambda_log_group +} + +resource "aws_cloudwatch_log_stream" "measurement-validation-agent-lambda-log-stream" { + name = local.measurement_validation_agent_lambda_stream_name + log_group_name = aws_cloudwatch_log_group.measurement-validation-agent-lambda-log-group.name +} + +resource "aws_iam_role_policy" "measurement_validation_agent_access_policy" { + name = "measurement_validation_agent_lambda_access_policy" + role = aws_iam_role.measurement_validation_agent_lambda_iam.name + policy = <