From d835dbfe896c3c4e5930809c7250a3260d1f6290 Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Sat, 27 Jul 2024 16:09:07 -0700 Subject: [PATCH 1/7] add setting, monitoring and expiring permissions --- break_glass/autokitteh.yaml | 7 +++++ break_glass/program.py | 59 +++++++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/break_glass/autokitteh.yaml b/break_glass/autokitteh.yaml index 2994cea1..c87ad007 100644 --- a/break_glass/autokitteh.yaml +++ b/break_glass/autokitteh.yaml @@ -6,8 +6,15 @@ project: - name: APPROVAL_CHANNEL value: "sre-team" connections: + - name: aws_connection + integration: aws - name: jira_connection integration: jira + - name: redis_connection + integration: redis + vars: + - name: URL + value: redis://localhost:6379/0 # Modify this if needed. - name: slack_connection integration: slack triggers: diff --git a/break_glass/program.py b/break_glass/program.py index ea5cc941..7bacf6c1 100644 --- a/break_glass/program.py +++ b/break_glass/program.py @@ -21,17 +21,24 @@ throughout the process. """ +from collections import namedtuple +from datetime import datetime import os from pathlib import Path +import time import autokitteh from autokitteh.atlassian import atlassian_jira_client +from autokitteh.aws import boto3_client + +from autokitteh.redis import redis_client from autokitteh.slack import slack_client from requests.exceptions import HTTPError APPROVAL_CHANNEL = os.getenv("APPROVAL_CHANNEL") jira = atlassian_jira_client("jira_connection") +redis = redis_client("redis_connection") slack = slack_client("slack_connection") @@ -42,7 +49,6 @@ def on_slack_slash_command(event): slack.views_open(trigger_id=trigger_id, view=request_modal) -@autokitteh.activity def on_form_submit(event): reason, issue_key, base_url, requester_id = parse_event_data(event) @@ -62,22 +68,27 @@ def on_form_submit(event): slack.chat_postMessage(channel=requester_id, text="Request sent for approval.") -@autokitteh.activity def on_approve_deny(event): + """Processes the approval/denial of the request and notifies the requester.""" action_id = event.data["actions"][0]["action_id"] _, requester, issue_key = action_id.split(" ") approver_id = event.data["user"]["id"] approver_info = slack.users_info(user=approver_id) - if event.data["actions"][0]["value"] == "Approve": - approver_email = approver_info["user"]["profile"]["email"] - jira.issue_add_comment(issue_key, f"Request approved by: {approver_email}") - message = f"Request approved by: <@{approver_info['user']['name']}>" - slack.chat_postMessage(channel=requester, text=message) - else: - print(f"Requester: {requester}") + if event.data["actions"][0]["value"] != "Approve": message = f"Request denied by: <@{approver_info["user"]["name"]}>" slack.chat_postMessage(channel=requester, text=message) + return + + approver_email = approver_info["user"]["profile"]["email"] + jira.issue_add_comment(issue_key, f"Request approved by: {approver_email}") + message = f"Request approved by: <@{approver_info['user']['name']}>" + slack.chat_postMessage(channel=requester, text=message) + + # TODO: get user from google sheets + aws_user = "break-glass-test-user" + set_permissions(aws_user) + monitor_and_remove_permissions(aws_user, requester) def send_approval_request(reason, issue_key, base_url, requester_id): @@ -94,6 +105,18 @@ def send_approval_request(reason, issue_key, base_url, requester_id): slack.chat_postMessage(channel=APPROVAL_CHANNEL, blocks=blocks) +def monitor_and_remove_permissions(aws_user, slack_user): + while True: + timestamp = get_user_timestamp(aws_user) + if not timestamp: + return + if float(timestamp) < time.time(): + break + time.sleep(10) + expire_permissions(aws_user) + slack.chat_postMessage(channel=slack_user, text="Your permissions have expired.") + + def parse_event_data(event): form_data = event.data["view"]["state"]["values"] reason = form_data["block_reason"]["reason"]["value"] @@ -107,6 +130,7 @@ def check_issue_exists(issue_key): try: jira.issue(issue_key) return True + # TODO: issue exists or a more specific error code 404 etc except HTTPError as e: print(f"Error retrieving issue: {e}") return False @@ -116,3 +140,20 @@ def validate_requester(issue_key, requester): issue = jira.issue(issue_key) assignee = issue.get("fields", {}).get("assignee", {}).get("emailAddress", "") return assignee == requester + + +@autokitteh.activity +def set_permissions(user_name): + aws.add_user_to_group(GroupName="break-glass-admin", UserName=user_name) + redis.set(user_name, time.time() + 120) + + +@autokitteh.activity +def expire_permissions(user_name): + aws.remove_user_from_group(GroupName="break-glass-admin", UserName=user_name) + redis.delete(user_name) + + +@autokitteh.activity +def get_user_timestamp(user_name): + return redis.get(user_name) From c456d6e68a3d6f763a8ecba59b0d237c2ec937a3 Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Mon, 29 Jul 2024 10:40:46 -0700 Subject: [PATCH 2/7] add permission expire global var --- break_glass/program.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/break_glass/program.py b/break_glass/program.py index 7bacf6c1..35d1ac98 100644 --- a/break_glass/program.py +++ b/break_glass/program.py @@ -126,6 +126,13 @@ def parse_event_data(event): return reason, issue_key, base_url, requester_id +def validate_requester(issue_key, requester): + issue = jira.issue(issue_key) + assignee = issue.get("fields", {}).get("assignee", {}).get("emailAddress", "") + return assignee == requester + + +@autokitteh.activity def check_issue_exists(issue_key): try: jira.issue(issue_key) @@ -136,16 +143,10 @@ def check_issue_exists(issue_key): return False -def validate_requester(issue_key, requester): - issue = jira.issue(issue_key) - assignee = issue.get("fields", {}).get("assignee", {}).get("emailAddress", "") - return assignee == requester - - @autokitteh.activity def set_permissions(user_name): aws.add_user_to_group(GroupName="break-glass-admin", UserName=user_name) - redis.set(user_name, time.time() + 120) + redis.set(user_name, time.time() + os.getenv("PERMISSION_EXPIRY")) @autokitteh.activity From 73b3e62969aa573f8e67268988cf2a09f1a63f5b Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Mon, 29 Jul 2024 16:40:29 -0700 Subject: [PATCH 3/7] add google sheets --- break_glass/autokitteh.yaml | 10 +++++++++- break_glass/program.py | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/break_glass/autokitteh.yaml b/break_glass/autokitteh.yaml index c87ad007..b6c967a0 100644 --- a/break_glass/autokitteh.yaml +++ b/break_glass/autokitteh.yaml @@ -4,10 +4,18 @@ project: name: break_glass vars: - name: APPROVAL_CHANNEL - value: "sre-team" + value: sre-team + - name: PERMISSION_EXPIRY + value: 60 # seconds + - name: SHEETS_RANGE + value: Sheet1!A:B + - name: SHEETS_ID + value: 14HTDDXapk548dBA1nNbeKq9NaXMsI7UE6Y2YkxQDjZs connections: - name: aws_connection integration: aws + - name: googlesheets_connection + integration: googlesheets - name: jira_connection integration: jira - name: redis_connection diff --git a/break_glass/program.py b/break_glass/program.py index 35d1ac98..4449b56c 100644 --- a/break_glass/program.py +++ b/break_glass/program.py @@ -21,8 +21,6 @@ throughout the process. """ -from collections import namedtuple -from datetime import datetime import os from pathlib import Path import time @@ -30,13 +28,17 @@ import autokitteh from autokitteh.atlassian import atlassian_jira_client from autokitteh.aws import boto3_client - +from autokitteh.google import google_sheets_client from autokitteh.redis import redis_client from autokitteh.slack import slack_client from requests.exceptions import HTTPError APPROVAL_CHANNEL = os.getenv("APPROVAL_CHANNEL") +SHEET_ID = os.getenv("SHEETS_ID") +SHEETS_RANGE = os.getenv("SHEETS_RANGE") +aws = boto3_client("aws_connection") +google_sheets = google_sheets_client("google_sheets_connection") jira = atlassian_jira_client("jira_connection") redis = redis_client("redis_connection") slack = slack_client("slack_connection") @@ -86,6 +88,10 @@ def on_approve_deny(event): slack.chat_postMessage(channel=requester, text=message) # TODO: get user from google sheets + sheet = google_sheets.spreadsheets() + result = sheet.values() + values = result.get(spreadsheetId=SHEET_ID, range=SHEETS_RANGE).execute() + print(values) aws_user = "break-glass-test-user" set_permissions(aws_user) monitor_and_remove_permissions(aws_user, requester) @@ -128,8 +134,11 @@ def parse_event_data(event): def validate_requester(issue_key, requester): issue = jira.issue(issue_key) - assignee = issue.get("fields", {}).get("assignee", {}).get("emailAddress", "") - return assignee == requester + assignee = issue.get("fields", {}).get("assignee", {}) + if assignee is None: + return False + email = assignee.get("emailAddress", "") + return email == requester @autokitteh.activity From f25385d3a41d249c113066c0daaf9c8bd142d44d Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Mon, 29 Jul 2024 16:40:36 -0700 Subject: [PATCH 4/7] add readme --- break_glass/README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 break_glass/README.md diff --git a/break_glass/README.md b/break_glass/README.md new file mode 100644 index 00000000..b98289fb --- /dev/null +++ b/break_glass/README.md @@ -0,0 +1,42 @@ +# Break Glass Request and Approval Orchestration + +This program orchestrates the request and approval process for break glass scenarios, where a developer needs elevated permissions to access sensitive data or perform critical operations beyond their usual access. + +## Benefits + +1. **Controlled Access**: Ensures sensitive data and operations are only accessible when necessary and with proper approval. +2. **Audit Trail**: Maintains a clear record of who requested and approved access, providing transparency and accountability. +3. **Automated Workflow**: Streamlines the request and approval process through automation, reducing manual overhead and potential for errors. + +## How It Works + +1. **Request Initiation**: + - A developer initiates the process using a Slack slash command to request break glass approval. + +2. **Information Gathering**: + - AutoKitteh sends a form to the developer, requesting details about the reason for the elevated access. + - The developer fills out the form, providing the necessary information and justification. + +3. **Verification**: + - The program integrates with Jira to verify the existence of the ticket and ensure the requester is the ticket's assignee. + +4. **Approval Request**: + - AutoKitteh sends a notification to the Site Reliability Engineering (SRE) team with an approve/deny message, including the details of the request. + - The SRE team reviews the request and makes a decision. + +5. **Notification**: + - AutoKitteh notifies the developer of the decision via Slack, indicating whether the request was approved or denied. + +6. **Permission Management**: + - If approved, permissions are granted to the developer, with a set expiration time. + - The system monitors the permissions and automatically removes them once they expire, ensuring minimal risk. + +7. **Integration with Services**: + - The program uses Slack for communication and notifications. + - It also integrates with AWS for permission management, Redis for storing timestamps, and Google Sheets for tracking purposes. + +## Usage + +To use this program, ensure you have the necessary environment variables set, such as `APPROVAL_CHANNEL`, `SHEET_ID`, `SHEETS_RANGE`, and other connection details for AWS, Google Sheets, Jira, Redis, and Slack. + +The workflow begins when a developer uses the Slack slash command to initiate a break glass request. The program then follows the outlined steps to ensure controlled, auditable access to sensitive resources. From 2f75e38760d968b21b565729cfeffca6ee2e2634 Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Tue, 30 Jul 2024 12:04:36 -0700 Subject: [PATCH 5/7] remove google for now and add global variables --- break_glass/autokitteh.yaml | 28 +++++++++++++++++++--------- break_glass/program.py | 32 ++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/break_glass/autokitteh.yaml b/break_glass/autokitteh.yaml index b6c967a0..43ae3e7e 100644 --- a/break_glass/autokitteh.yaml +++ b/break_glass/autokitteh.yaml @@ -1,27 +1,37 @@ +# This YAML file is a declarative manifest that describes the setup +# of the AutoKitteh project "Break Glass". +# Break Glass integrates AWS, Jira, Redis, and Slack to streamline +# emergency access requests. +# +# Before applying this file: +# - Modify the values in the project's "vars" section, if desired +# - Modify the Redis connection string, if needed +# +# After applying this file, initialize this AutoKitteh project's +# AWS, Jira and Slack connections. + version: v1 project: name: break_glass vars: - name: APPROVAL_CHANNEL - value: sre-team + value: sre-team # Modify this if needed. + - name: AWS_ADMIN_GROUP + value: break-glass-admin # Modify this if needed. + - name: REQUESTER_EMAIL # can be replaced with a database or spreadsheet lookup + value: pasha@autokitteh.com - name: PERMISSION_EXPIRY - value: 60 # seconds - - name: SHEETS_RANGE - value: Sheet1!A:B - - name: SHEETS_ID - value: 14HTDDXapk548dBA1nNbeKq9NaXMsI7UE6Y2YkxQDjZs + value: 10 # seconds connections: - name: aws_connection integration: aws - - name: googlesheets_connection - integration: googlesheets - name: jira_connection integration: jira - name: redis_connection integration: redis vars: - - name: URL + - name: url value: redis://localhost:6379/0 # Modify this if needed. - name: slack_connection integration: slack diff --git a/break_glass/program.py b/break_glass/program.py index 4449b56c..6b16acbf 100644 --- a/break_glass/program.py +++ b/break_glass/program.py @@ -35,10 +35,10 @@ APPROVAL_CHANNEL = os.getenv("APPROVAL_CHANNEL") -SHEET_ID = os.getenv("SHEETS_ID") -SHEETS_RANGE = os.getenv("SHEETS_RANGE") -aws = boto3_client("aws_connection") -google_sheets = google_sheets_client("google_sheets_connection") +AWS_ADMIN_GROUP = os.getenv("AWS_ADMIN_GROUP") +PERMISSION_EXPIRY = os.getenv("PERMISSION_EXPIRY") +REQUESTER_EMAIL = os.getenv("REQUESTER_EMAIL") +aws = boto3_client("aws_connection", "iam") jira = atlassian_jira_client("jira_connection") redis = redis_client("redis_connection") slack = slack_client("slack_connection") @@ -87,12 +87,7 @@ def on_approve_deny(event): message = f"Request approved by: <@{approver_info['user']['name']}>" slack.chat_postMessage(channel=requester, text=message) - # TODO: get user from google sheets - sheet = google_sheets.spreadsheets() - result = sheet.values() - values = result.get(spreadsheetId=SHEET_ID, range=SHEETS_RANGE).execute() - print(values) - aws_user = "break-glass-test-user" + aws_user = lookup_user_by_email(approver_email) set_permissions(aws_user) monitor_and_remove_permissions(aws_user, requester) @@ -141,12 +136,20 @@ def validate_requester(issue_key, requester): return email == requester +def lookup_user_by_email(email): + # this is meant to be a placeholder for a real lookup function + email_to_user = { + REQUESTER_EMAIL: "break-glass-test-user", + "david@example.com": "random-user", + } + return email_to_user.get(email) + + @autokitteh.activity def check_issue_exists(issue_key): try: jira.issue(issue_key) return True - # TODO: issue exists or a more specific error code 404 etc except HTTPError as e: print(f"Error retrieving issue: {e}") return False @@ -154,13 +157,14 @@ def check_issue_exists(issue_key): @autokitteh.activity def set_permissions(user_name): - aws.add_user_to_group(GroupName="break-glass-admin", UserName=user_name) - redis.set(user_name, time.time() + os.getenv("PERMISSION_EXPIRY")) + aws.add_user_to_group(GroupName=AWS_ADMIN_GROUP, UserName=user_name) + time_alotted = os.getenv("PERMISSION_EXPIRY") + redis.set(user_name, time.time() + float(time_alotted)) @autokitteh.activity def expire_permissions(user_name): - aws.remove_user_from_group(GroupName="break-glass-admin", UserName=user_name) + aws.remove_user_from_group(GroupName=AWS_ADMIN_GROUP, UserName=user_name) redis.delete(user_name) From f93bbda15defc1bc821284d52518cfa3b6cc9065 Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Tue, 30 Jul 2024 12:14:15 -0700 Subject: [PATCH 6/7] remove personal information --- break_glass/autokitteh.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/break_glass/autokitteh.yaml b/break_glass/autokitteh.yaml index 43ae3e7e..120492fe 100644 --- a/break_glass/autokitteh.yaml +++ b/break_glass/autokitteh.yaml @@ -16,11 +16,11 @@ project: name: break_glass vars: - name: APPROVAL_CHANNEL - value: sre-team # Modify this if needed. + value: # Modify this if needed. - name: AWS_ADMIN_GROUP value: break-glass-admin # Modify this if needed. - name: REQUESTER_EMAIL # can be replaced with a database or spreadsheet lookup - value: pasha@autokitteh.com + value: - name: PERMISSION_EXPIRY value: 10 # seconds connections: From a948054e635e9d211e21bdf66f6ed19730f7b23a Mon Sep 17 00:00:00 2001 From: Pavel Fateev Date: Tue, 30 Jul 2024 13:25:33 -0700 Subject: [PATCH 7/7] remove unused import --- break_glass/program.py | 1 - 1 file changed, 1 deletion(-) diff --git a/break_glass/program.py b/break_glass/program.py index 6b16acbf..542b422a 100644 --- a/break_glass/program.py +++ b/break_glass/program.py @@ -28,7 +28,6 @@ import autokitteh from autokitteh.atlassian import atlassian_jira_client from autokitteh.aws import boto3_client -from autokitteh.google import google_sheets_client from autokitteh.redis import redis_client from autokitteh.slack import slack_client from requests.exceptions import HTTPError