From 0685dcb96baa25a5bc7c2b3728c59f16ef6d7f4e Mon Sep 17 00:00:00 2001 From: alonit Date: Wed, 20 Jul 2022 12:59:37 +0300 Subject: [PATCH 01/32] microservices architecture --- bot.py | 50 ++++++++++++++++++++++++++++++++++++++++-------- config.json | 5 +++++ requirements.txt | 2 +- utils.py | 22 ++++++++++++++++++++- worker.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 config.json create mode 100644 worker.py diff --git a/bot.py b/bot.py index 7e311287..29baf126 100644 --- a/bot.py +++ b/bot.py @@ -1,6 +1,10 @@ +import json +import threading +import botocore from telegram.ext import Updater, MessageHandler, Filters -from utils import search_download_youtube_video from loguru import logger +import boto3 +from utils import calc_backoff_per_instance class Bot: @@ -26,10 +30,13 @@ def send_video(self, update, context, file_path): """Sends video to a chat""" context.bot.send_video(chat_id=update.message.chat_id, video=open(file_path, 'rb'), supports_streaming=True) - def send_text(self, update, text, quote=False): + def send_text(self, update, text, chat_id=None, quote=False): """Sends text to a chat""" - # retry https://github.com/python-telegram-bot/python-telegram-bot/issues/1124 - update.message.reply_text(text, quote=quote) + if chat_id: + self.updater.bot.send_message(chat_id, text=text) + else: + # retry https://github.com/python-telegram-bot/python-telegram-bot/issues/1124 + update.message.reply_text(text, quote=quote) class QuoteBot(Bot): @@ -42,14 +49,41 @@ def _message_handler(self, update, context): self.send_text(update, f'Your original message: {update.message.text}', quote=to_quote) -class YoutubeBot(Bot): - pass +class YoutubeObjectDetectBot(Bot): + def __init__(self, token): + super().__init__(token) + threading.Thread( + target=calc_backoff_per_instance, + args=(workers_queue, asg, config.get("autoscaling_group_name")) + ).start() + + def _message_handler(self, update, context): + try: + chat_id = str(update.effective_message.chat_id) + response = workers_queue.send_message( + MessageBody=update.message.text, + MessageAttributes={ + 'chat_id': {'StringValue': chat_id, 'DataType': 'String'} + } + ) + logger.info(f'msg {response.get("MessageId")} has been sent to queue') + self.send_text(update, f'Your message is being processed...', chat_id=chat_id) + + except botocore.exceptions.ClientError as error: + logger.error(error) + self.send_text(update, f'Something went wrong, please try again...') if __name__ == '__main__': with open('.telegramToken') as f: _token = f.read() - my_bot = Bot(_token) - my_bot.start() + with open('config.json') as f: + config = json.load(f) + + sqs = boto3.resource('sqs', region_name=config.get('aws_region')) + workers_queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) + asg = boto3.client('autoscaling', region_name=config.get('aws_region')) + my_bot = YoutubeObjectDetectBot(_token) + my_bot.start() diff --git a/config.json b/config.json new file mode 100644 index 00000000..75538aca --- /dev/null +++ b/config.json @@ -0,0 +1,5 @@ +{ + "aws_region": "region-id", + "bot_to_worker_queue_name": "queue-name", + "autoscaling_group_name": "asg-name" +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 28cc6ee3..ad486b56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ python-telegram-bot>=13.11 -youtube-dl>=2021.12.17 +yt-dlp>=2022.6.29 loguru \ No newline at end of file diff --git a/utils.py b/utils.py index 0dcbc25c..cc1d8df7 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,6 @@ -from youtube_dl import YoutubeDL +import time +from yt_dlp import YoutubeDL +from loguru import logger def search_download_youtube_video(video_name, num_results=1): @@ -13,3 +15,21 @@ def search_download_youtube_video(video_name, num_results=1): return [ydl.prepare_filename(video) for video in videos] + +def calc_backoff_per_instance(sqs_queue_client, asg_client, asg_group_name): + while True: + msgs_in_queue = int(sqs_queue_client.attributes.get('ApproximateNumberOfMessages')) + asg_size = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_group_name])['AutoScalingGroups'][0]['DesiredCapacity'] + + if msgs_in_queue == 0: + backoff_per_instance = 0 + elif asg_size == 0: + backoff_per_instance = 99 + else: + backoff_per_instance = msgs_in_queue / asg_size + + logger.info(f'backoff per instance: {backoff_per_instance}') + + # TODO send the backoff_per_instance metric to cloudwatch + + time.sleep(60) diff --git a/worker.py b/worker.py new file mode 100644 index 00000000..23881ea1 --- /dev/null +++ b/worker.py @@ -0,0 +1,47 @@ +import json +import time +import boto3 +import botocore +from loguru import logger +from utils import search_download_youtube_video + + +def process_msg(msg): + search_download_youtube_video(msg) + + # TODO upload the downloaded video to your S3 bucket + + +def main(): + while True: + try: + messages = queue.receive_messages( + MessageAttributeNames=['All'], + MaxNumberOfMessages=1, + WaitTimeSeconds=10 + ) + for msg in messages: + logger.info(f'processing message {msg}') + process_msg(msg.body) + + # delete message from the queue after is was handled + response = queue.delete_messages(Entries=[{ + 'Id': msg.message_id, + 'ReceiptHandle': msg.receipt_handle + }]) + if 'Successful' in response: + logger.info(f'msg {msg} has been handled successfully') + + except botocore.exceptions.ClientError as err: + logger.exception(f"Couldn't receive messages {err}") + time.sleep(10) + + +if __name__ == '__main__': + with open('config.json') as f: + config = json.load(f) + + sqs = boto3.resource('sqs', region_name=config.get('aws_region')) + queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) + + main() From aadefa8d2568362acca3699822b2580ca45b1d23 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 19:36:22 +0300 Subject: [PATCH 02/32] . --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 75538aca..e1094cfe 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,5 @@ { - "aws_region": "region-id", - "bot_to_worker_queue_name": "queue-name", + "aws_region": "eu-central-1", + "bot_to_worker_queue_name": "daniel-reuven-aws-ex1-polybot-queue", "autoscaling_group_name": "asg-name" } \ No newline at end of file From e5fbaf4ec9a1385d6369eed39bad18ff78b92e55 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 21:12:51 +0300 Subject: [PATCH 03/32] update 1 --- .gitignore | 3 ++ bot.py | 4 +- config.json | 4 +- requirements.txt | 4 +- utils.py | 111 +++++++++++++++++++++++++++++++++++++++++------ worker.py | 8 ++-- 6 files changed, 112 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index bf0d076b..7c233c96 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ __pycache__/ # C extensions *.so +# Telegram stuff +*.telegramFile +*.telegramToken # ignore .pem file *.pem diff --git a/bot.py b/bot.py index 29baf126..1ad8c75b 100644 --- a/bot.py +++ b/bot.py @@ -4,7 +4,7 @@ from telegram.ext import Updater, MessageHandler, Filters from loguru import logger import boto3 -from utils import calc_backoff_per_instance +from utils import calc_backlog_per_instance class Bot: @@ -53,7 +53,7 @@ class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) threading.Thread( - target=calc_backoff_per_instance, + target=calc_backlog_per_instance, args=(workers_queue, asg, config.get("autoscaling_group_name")) ).start() diff --git a/config.json b/config.json index e1094cfe..a6e51f92 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,7 @@ { "aws_region": "eu-central-1", "bot_to_worker_queue_name": "daniel-reuven-aws-ex1-polybot-queue", - "autoscaling_group_name": "asg-name" + "autoscaling_group_name": "asg-name", + "cloudwatch_namespace": "cloudwatch-namespace", + "bucket_name": "daniel-reuven-aws-ex1-polybot-bucket" } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ad486b56..c9358c3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ python-telegram-bot>=13.11 yt-dlp>=2022.6.29 -loguru \ No newline at end of file +loguru~=0.6.0 +botocore~=1.27.13 +boto3~=1.24.13 \ No newline at end of file diff --git a/utils.py b/utils.py index cc1d8df7..e8855d80 100644 --- a/utils.py +++ b/utils.py @@ -1,35 +1,120 @@ +# import logging import time +import os +import boto3 +import botocore +from botocore.exceptions import ClientError from yt_dlp import YoutubeDL from loguru import logger +from time import sleep -def search_download_youtube_video(video_name, num_results=1): +def search_download_youtube_video(video_name, num_results, s3_bucket_name): """ This function downloads the first num_results search results from Youtube + :param s3_bucket_name: string of the S3 bucket name :param video_name: string of the video name :param num_results: integer representing how many videos to download :return: list of paths to your downloaded video files """ - with YoutubeDL() as ydl: - videos = ydl.extract_info(f"ytsearch{num_results}:{video_name}", download=True)['entries'] + # Parameters for youtube_dl use + ydl = {'noplaylist': 'True', 'format': 'bestvideo[ext=mp4]+bestaudio[ext=mp4]/mp4', 'outtmpl': '/./ytdlAppData/%(id)s.%(ext)s'} + # Try to download and return list of video/s or error msg + with YoutubeDL(ydl) as ydl: + ydl.cache.remove() + # 1. get a list of video file names with download=false parameter + videos = ydl.extract_info(f"ytsearch{num_results}:{video_name}", download=False)['entries'] + for video in videos: + # if video['duration'] >= 900: + # return "Error, selected track/s are above predefined duration limit" + # if video['duration'] <= 0.1: + # return "Error, selected track/s are below predefined duration limit" + prefix = 'ytdlAppData/' + video['id'] + '.mp4' + # print(prefix) + # check aws s3 bucket for file, then locally and act accordingly,prefix != ydl.prepare_filename(video) + if not (check_s3_file(prefix, s3_bucket_name)): + if not (os.path.isfile(ydl.prepare_filename(video))): + video_url = video['webpage_url'] + ydl.extract_info(video_url, download=True) + upload_file(prefix, s3_bucket_name) + os.remove(ydl.prepare_filename(video)) + else: + upload_file(prefix, s3_bucket_name) + os.remove(ydl.prepare_filename(video)) + else: + if os.path.isfile(ydl.prepare_filename(video)): + # download_file(prefix, s3_bucket_name) + os.remove(ydl.prepare_filename(video)) + sleep(1) + return [ydl.prepare_filename(video) for video in videos] - return [ydl.prepare_filename(video) for video in videos] - -def calc_backoff_per_instance(sqs_queue_client, asg_client, asg_group_name): +def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): while True: msgs_in_queue = int(sqs_queue_client.attributes.get('ApproximateNumberOfMessages')) asg_size = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_group_name])['AutoScalingGroups'][0]['DesiredCapacity'] - if msgs_in_queue == 0: - backoff_per_instance = 0 + backlog_per_instance = 0 elif asg_size == 0: - backoff_per_instance = 99 + backlog_per_instance = 99 + else: + backlog_per_instance = msgs_in_queue / asg_size + logger.info(f'backlog per instance: {backlog_per_instance}') + # Create CloudWatch client + cloudwatch = boto3.client('cloudwatch') + # Put custom metrics + cloudwatch.put_metric_data( + Namespace='monitor-polybot-asg', + MetricData=[ + { + 'MetricName': 'backlog_per_instance', + 'Value': backlog_per_instance, + 'Unit': 'Count' + }, + ] + ) + time.sleep(60) + + +def check_s3_file(key_filename, s3_bucket_name): + s3 = boto3.resource('s3') + try: + s3.Object(s3_bucket_name, key_filename).load() + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == "404": + # The object does not exist. + return False else: - backoff_per_instance = msgs_in_queue / asg_size + # Something else has gone wrong. + raise + else: + # The object does exist. + return True - logger.info(f'backoff per instance: {backoff_per_instance}') - # TODO send the backoff_per_instance metric to cloudwatch +def upload_file(key_filename, bucket, object_name=None): + # Upload the file + s3_client = boto3.client('s3') + # If S3 object_name was not specified, use key_filename + if object_name is None: + object_name = key_filename + try: + response = s3_client.upload_file(key_filename, bucket, object_name) + except ClientError as e: + logger.error(e) + return False + return True - time.sleep(60) + +def download_file(key_filename, bucket, object_name=None): + # Upload the file + s3_client = boto3.client('s3') + # If S3 object_name was not specified, use key_filename + if object_name is None: + object_name = key_filename + try: + response = s3_client.download_file(bucket, object_name, key_filename) + except ClientError as e: + logger.error(e) + return False + return True diff --git a/worker.py b/worker.py index 23881ea1..492aa56d 100644 --- a/worker.py +++ b/worker.py @@ -7,9 +7,7 @@ def process_msg(msg): - search_download_youtube_video(msg) - - # TODO upload the downloaded video to your S3 bucket + search_download_youtube_video(msg, 1, s3_bucket_name) def main(): @@ -24,7 +22,7 @@ def main(): logger.info(f'processing message {msg}') process_msg(msg.body) - # delete message from the queue after is was handled + # delete message from the queue after it was handled response = queue.delete_messages(Entries=[{ 'Id': msg.message_id, 'ReceiptHandle': msg.receipt_handle @@ -43,5 +41,5 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) - + s3_bucket_name = config.get('bot_to_worker_queue_name') main() From 9304aed87212820d461a63c7fe93730003d5f46d Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 21:55:26 +0300 Subject: [PATCH 04/32] update 2, testing path and python download --- bot.py | 8 ++++---- worker.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/bot.py b/bot.py index 1ad8c75b..badfbc2d 100644 --- a/bot.py +++ b/bot.py @@ -52,10 +52,10 @@ def _message_handler(self, update, context): class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) - threading.Thread( - target=calc_backlog_per_instance, - args=(workers_queue, asg, config.get("autoscaling_group_name")) - ).start() + # threading.Thread( + # target=calc_backlog_per_instance, + # args=(workers_queue, asg, config.get("autoscaling_group_name")) + # ).start() def _message_handler(self, update, context): try: diff --git a/worker.py b/worker.py index 492aa56d..ba997f20 100644 --- a/worker.py +++ b/worker.py @@ -2,6 +2,7 @@ import time import boto3 import botocore +import os from loguru import logger from utils import search_download_youtube_video @@ -42,4 +43,15 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) s3_bucket_name = config.get('bot_to_worker_queue_name') + + cwd = os.getcwd() + path = f"{cwd}/ytdlAppData" + # Check whether the specified path exists or not + isExist = os.path.exists(path) + + if not isExist: + # Create a new directory because it does not exist + os.makedirs(path) + print("The new directory is created!") + main() From a583d29b0b17759cc1ed84516b9f8dc7bec65c5f Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 22:08:03 +0300 Subject: [PATCH 05/32] update 2, testing path and python download 2 --- utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils.py b/utils.py index e8855d80..5e8efeb0 100644 --- a/utils.py +++ b/utils.py @@ -18,7 +18,7 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): :return: list of paths to your downloaded video files """ # Parameters for youtube_dl use - ydl = {'noplaylist': 'True', 'format': 'bestvideo[ext=mp4]+bestaudio[ext=mp4]/mp4', 'outtmpl': '/./ytdlAppData/%(id)s.%(ext)s'} + ydl = {'noplaylist': 'True', 'format': 'bestvideo[ext=mp4]+bestaudio[ext=mp4]/mp4', 'outtmpl': '%(id)s.%(ext)s'} # Try to download and return list of video/s or error msg with YoutubeDL(ydl) as ydl: ydl.cache.remove() @@ -29,6 +29,7 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): # return "Error, selected track/s are above predefined duration limit" # if video['duration'] <= 0.1: # return "Error, selected track/s are below predefined duration limit" + localprefix = video['id'] + '.mp4' prefix = 'ytdlAppData/' + video['id'] + '.mp4' # print(prefix) # check aws s3 bucket for file, then locally and act accordingly,prefix != ydl.prepare_filename(video) @@ -36,10 +37,10 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): if not (os.path.isfile(ydl.prepare_filename(video))): video_url = video['webpage_url'] ydl.extract_info(video_url, download=True) - upload_file(prefix, s3_bucket_name) + upload_file(localprefix, s3_bucket_name) os.remove(ydl.prepare_filename(video)) else: - upload_file(prefix, s3_bucket_name) + upload_file(localprefix, s3_bucket_name) os.remove(ydl.prepare_filename(video)) else: if os.path.isfile(ydl.prepare_filename(video)): From 3b47eafa3485645e30ec75bd76573f86355cef01 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 22:13:11 +0300 Subject: [PATCH 06/32] update 2, testing path and python download 3 --- worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.py b/worker.py index ba997f20..22fa598c 100644 --- a/worker.py +++ b/worker.py @@ -42,7 +42,7 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) - s3_bucket_name = config.get('bot_to_worker_queue_name') + s3_bucket_name = config.get('bucket_name') cwd = os.getcwd() path = f"{cwd}/ytdlAppData" From 80220d7b63126f5a06d52df0a8544c71340fcd86 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 22:29:03 +0300 Subject: [PATCH 07/32] update 2, testing path and python download 4 --- utils.py | 2 +- worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 5e8efeb0..64e50010 100644 --- a/utils.py +++ b/utils.py @@ -31,7 +31,7 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): # return "Error, selected track/s are below predefined duration limit" localprefix = video['id'] + '.mp4' prefix = 'ytdlAppData/' + video['id'] + '.mp4' - # print(prefix) + print(s3_bucket_name) # check aws s3 bucket for file, then locally and act accordingly,prefix != ydl.prepare_filename(video) if not (check_s3_file(prefix, s3_bucket_name)): if not (os.path.isfile(ydl.prepare_filename(video))): diff --git a/worker.py b/worker.py index 22fa598c..31eac592 100644 --- a/worker.py +++ b/worker.py @@ -43,7 +43,7 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) s3_bucket_name = config.get('bucket_name') - + print(s3_bucket_name) cwd = os.getcwd() path = f"{cwd}/ytdlAppData" # Check whether the specified path exists or not From 98f86826024b972c79ad86ed7096d69dae6dffb4 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 22:33:22 +0300 Subject: [PATCH 08/32] update 2, testing path and python download 5 --- utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/utils.py b/utils.py index 64e50010..350ed41f 100644 --- a/utils.py +++ b/utils.py @@ -31,7 +31,7 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): # return "Error, selected track/s are below predefined duration limit" localprefix = video['id'] + '.mp4' prefix = 'ytdlAppData/' + video['id'] + '.mp4' - print(s3_bucket_name) + # print(s3_bucket_name) # check aws s3 bucket for file, then locally and act accordingly,prefix != ydl.prepare_filename(video) if not (check_s3_file(prefix, s3_bucket_name)): if not (os.path.isfile(ydl.prepare_filename(video))): @@ -78,9 +78,10 @@ def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): def check_s3_file(key_filename, s3_bucket_name): + s3_prefix = 'ytdlAppData/' + key_filename s3 = boto3.resource('s3') try: - s3.Object(s3_bucket_name, key_filename).load() + s3.Object(s3_bucket_name, s3_prefix).load() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": # The object does not exist. @@ -94,13 +95,14 @@ def check_s3_file(key_filename, s3_bucket_name): def upload_file(key_filename, bucket, object_name=None): + s3_prefix = 'ytdlAppData/' + key_filename # Upload the file s3_client = boto3.client('s3') # If S3 object_name was not specified, use key_filename if object_name is None: - object_name = key_filename + object_name = s3_prefix try: - response = s3_client.upload_file(key_filename, bucket, object_name) + response = s3_client.upload_file(s3_prefix, bucket, object_name) except ClientError as e: logger.error(e) return False @@ -108,13 +110,14 @@ def upload_file(key_filename, bucket, object_name=None): def download_file(key_filename, bucket, object_name=None): + s3_prefix = 'ytdlAppData/' + key_filename # Upload the file s3_client = boto3.client('s3') # If S3 object_name was not specified, use key_filename if object_name is None: - object_name = key_filename + object_name = s3_prefix try: - response = s3_client.download_file(bucket, object_name, key_filename) + response = s3_client.download_file(bucket, object_name, s3_prefix) except ClientError as e: logger.error(e) return False From ad7b3b5776adc090d6c975c972db905f98cb6c78 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 20 Jul 2022 22:38:46 +0300 Subject: [PATCH 09/32] update 2, testing path and python download 6 --- utils.py | 2 +- worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 350ed41f..7b173d0b 100644 --- a/utils.py +++ b/utils.py @@ -102,7 +102,7 @@ def upload_file(key_filename, bucket, object_name=None): if object_name is None: object_name = s3_prefix try: - response = s3_client.upload_file(s3_prefix, bucket, object_name) + response = s3_client.upload_file(key_filename, bucket, s3_prefix) except ClientError as e: logger.error(e) return False diff --git a/worker.py b/worker.py index 31eac592..51e9ac5e 100644 --- a/worker.py +++ b/worker.py @@ -43,7 +43,7 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) s3_bucket_name = config.get('bucket_name') - print(s3_bucket_name) + # print(s3_bucket_name) cwd = os.getcwd() path = f"{cwd}/ytdlAppData" # Check whether the specified path exists or not From 2adabf0ddb3686fdc86d6f5bf8a5d1d2b6b52fe7 Mon Sep 17 00:00:00 2001 From: alonit Date: Thu, 21 Jul 2022 13:49:48 +0300 Subject: [PATCH 10/32] typo --- .gitignore | 1 + bot.py | 4 ++-- utils.py | 12 ++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index bf0d076b..326440d2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # C extensions *.so +.telegramToken # ignore .pem file *.pem diff --git a/bot.py b/bot.py index 29baf126..1ad8c75b 100644 --- a/bot.py +++ b/bot.py @@ -4,7 +4,7 @@ from telegram.ext import Updater, MessageHandler, Filters from loguru import logger import boto3 -from utils import calc_backoff_per_instance +from utils import calc_backlog_per_instance class Bot: @@ -53,7 +53,7 @@ class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) threading.Thread( - target=calc_backoff_per_instance, + target=calc_backlog_per_instance, args=(workers_queue, asg, config.get("autoscaling_group_name")) ).start() diff --git a/utils.py b/utils.py index cc1d8df7..c6d256eb 100644 --- a/utils.py +++ b/utils.py @@ -16,20 +16,20 @@ def search_download_youtube_video(video_name, num_results=1): return [ydl.prepare_filename(video) for video in videos] -def calc_backoff_per_instance(sqs_queue_client, asg_client, asg_group_name): +def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): while True: msgs_in_queue = int(sqs_queue_client.attributes.get('ApproximateNumberOfMessages')) asg_size = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_group_name])['AutoScalingGroups'][0]['DesiredCapacity'] if msgs_in_queue == 0: - backoff_per_instance = 0 + backlog_per_instance = 0 elif asg_size == 0: - backoff_per_instance = 99 + backlog_per_instance = 99 else: - backoff_per_instance = msgs_in_queue / asg_size + backlog_per_instance = msgs_in_queue / asg_size - logger.info(f'backoff per instance: {backoff_per_instance}') + logger.info(f'backlog per instance: {backlog_per_instance}') - # TODO send the backoff_per_instance metric to cloudwatch + # TODO send the backlog_per_instance metric to cloudwatch time.sleep(60) From 4b68511005eb35fe1c1d0272fb1c0e3a8c0b90f1 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 23 Jul 2022 13:30:20 +0300 Subject: [PATCH 11/32] update 3, bot testing --- bot.py | 8 ++++---- config.json | 2 +- utils.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bot.py b/bot.py index badfbc2d..1ad8c75b 100644 --- a/bot.py +++ b/bot.py @@ -52,10 +52,10 @@ def _message_handler(self, update, context): class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) - # threading.Thread( - # target=calc_backlog_per_instance, - # args=(workers_queue, asg, config.get("autoscaling_group_name")) - # ).start() + threading.Thread( + target=calc_backlog_per_instance, + args=(workers_queue, asg, config.get("autoscaling_group_name")) + ).start() def _message_handler(self, update, context): try: diff --git a/config.json b/config.json index a6e51f92..549c5efc 100644 --- a/config.json +++ b/config.json @@ -1,7 +1,7 @@ { "aws_region": "eu-central-1", "bot_to_worker_queue_name": "daniel-reuven-aws-ex1-polybot-queue", - "autoscaling_group_name": "asg-name", + "autoscaling_group_name": "daniel-reuven-polybot-asg", "cloudwatch_namespace": "cloudwatch-namespace", "bucket_name": "daniel-reuven-aws-ex1-polybot-bucket" } \ No newline at end of file diff --git a/utils.py b/utils.py index 7b173d0b..210dcd4b 100644 --- a/utils.py +++ b/utils.py @@ -65,7 +65,7 @@ def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): cloudwatch = boto3.client('cloudwatch') # Put custom metrics cloudwatch.put_metric_data( - Namespace='monitor-polybot-asg', + Namespace='daniel-reuven-monitor-polybot-asg', MetricData=[ { 'MetricName': 'backlog_per_instance', From 35d09a48e64bd43aa6c4bc82a1d26e588bc50d1a Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 23 Jul 2022 13:55:59 +0300 Subject: [PATCH 12/32] update 3, bot testing 2 --- .gitignore | 1 + bot.py | 2 +- utils.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7c233c96..f37f683c 100644 --- a/.gitignore +++ b/.gitignore @@ -157,3 +157,4 @@ crashlytics.properties crashlytics-build.properties fabric.properties +/asg-config.json diff --git a/bot.py b/bot.py index 1ad8c75b..033c0f66 100644 --- a/bot.py +++ b/bot.py @@ -54,7 +54,7 @@ def __init__(self, token): super().__init__(token) threading.Thread( target=calc_backlog_per_instance, - args=(workers_queue, asg, config.get("autoscaling_group_name")) + args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) ).start() def _message_handler(self, update, context): diff --git a/utils.py b/utils.py index 210dcd4b..fb815cfa 100644 --- a/utils.py +++ b/utils.py @@ -50,7 +50,7 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): return [ydl.prepare_filename(video) for video in videos] -def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): +def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name, aws_region): while True: msgs_in_queue = int(sqs_queue_client.attributes.get('ApproximateNumberOfMessages')) asg_size = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_group_name])['AutoScalingGroups'][0]['DesiredCapacity'] @@ -62,7 +62,7 @@ def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name): backlog_per_instance = msgs_in_queue / asg_size logger.info(f'backlog per instance: {backlog_per_instance}') # Create CloudWatch client - cloudwatch = boto3.client('cloudwatch') + cloudwatch = boto3.client('cloudwatch', aws_region) # Put custom metrics cloudwatch.put_metric_data( Namespace='daniel-reuven-monitor-polybot-asg', From 6ca8a20bdc51b916ce90a966b97ad45b7fb9301a Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 27 Jul 2022 20:37:01 +0300 Subject: [PATCH 13/32] update 4, adding dynamodb --- bot.py | 24 +++++++++++++++++++++++- config.json | 3 ++- utils.py | 20 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/bot.py b/bot.py index 033c0f66..b755b4dd 100644 --- a/bot.py +++ b/bot.py @@ -4,7 +4,8 @@ from telegram.ext import Updater, MessageHandler, Filters from loguru import logger import boto3 -from utils import calc_backlog_per_instance +from boto3.dynamodb.conditions import Key +from utils import calc_backlog_per_instance, search_youtube_video class Bot: @@ -69,6 +70,25 @@ def _message_handler(self, update, context): logger.info(f'msg {response.get("MessageId")} has been sent to queue') self.send_text(update, f'Your message is being processed...', chat_id=chat_id) + if update.message.text.startswith('/myvideos'): + response = table.query(KeyConditionExpression=Key('chatId').eq(chat_id)) + for key, value in response.items(): + array_length = len(value) + for i in range(array_length): + temp_dict = value[i] + video_url = temp_dict['url'] + video = search_youtube_video(None, video_url) + self.send_text(update, f'Video title: {video["title"]}, Video Link: {video["webpage_url"]}', chat_id=chat_id) + else: + for video in search_youtube_video(update.message.text, None): + item = { + 'chatId': chat_id, + 'videoId': video['id'], + 'url': video['webpage_url'], + 'title': video['title'] + } + response2 = table.put_item(Item=item) + except botocore.exceptions.ClientError as error: logger.error(error) self.send_text(update, f'Something went wrong, please try again...') @@ -84,6 +104,8 @@ def _message_handler(self, update, context): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) workers_queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) asg = boto3.client('autoscaling', region_name=config.get('aws_region')) + dynamodb = boto3.resource('dynamodb', region_name=config.get('aws_region')) + table = dynamodb.Table(config.get('table_name')) my_bot = YoutubeObjectDetectBot(_token) my_bot.start() diff --git a/config.json b/config.json index 549c5efc..ecc59525 100644 --- a/config.json +++ b/config.json @@ -3,5 +3,6 @@ "bot_to_worker_queue_name": "daniel-reuven-aws-ex1-polybot-queue", "autoscaling_group_name": "daniel-reuven-polybot-asg", "cloudwatch_namespace": "cloudwatch-namespace", - "bucket_name": "daniel-reuven-aws-ex1-polybot-bucket" + "bucket_name": "daniel-reuven-aws-ex1-polybot-bucket", + "table_name": "daniel-reuven-aws-ex1-polybot-ddb-table1" } \ No newline at end of file diff --git a/utils.py b/utils.py index fb815cfa..194ac194 100644 --- a/utils.py +++ b/utils.py @@ -50,6 +50,26 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): return [ydl.prepare_filename(video) for video in videos] +def search_youtube_video(video_name, video_url): + """ + This function downloads the first num_results search results from YouTube + :param video_url: url of the video + :param video_name: string of the video name + :return: list of paths to your downloaded video files + """ + # Parameters for youtube_dl use + ydl = {'noplaylist': 'True', 'format': 'bestvideo[ext=mp4]+bestaudio[ext=mp4]/mp4', 'outtmpl': '%(id)s.%(ext)s'} + # Try to download and return list of video/s or error msg + with YoutubeDL(ydl) as ydl: + ydl.cache.remove() + if video_name is None: + videos = ydl.extract_info(video_url, download=False) + return videos + elif video_url is None: + videos = ydl.extract_info(f"ytsearch{1}:{video_name}", download=False)['entries'] + return videos + + def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name, aws_region): while True: msgs_in_queue = int(sqs_queue_client.attributes.get('ApproximateNumberOfMessages')) From bfeaf34bbc8769beebbfe7e6ee6ac6d5a843fa6f Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sun, 31 Jul 2022 20:31:35 +0300 Subject: [PATCH 14/32] update code with presigned link to S3 --- bot.py | 39 +++++++++++++++++------------ config.json | 1 + utils.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++---- worker.py | 19 +++++++++----- 4 files changed, 103 insertions(+), 27 deletions(-) diff --git a/bot.py b/bot.py index b755b4dd..157f84c7 100644 --- a/bot.py +++ b/bot.py @@ -5,7 +5,8 @@ from loguru import logger import boto3 from boto3.dynamodb.conditions import Key -from utils import calc_backlog_per_instance, search_youtube_video +from utils import calc_backlog_per_instance, search_youtube_video, send_videos_from_queue2 + class Bot: @@ -57,29 +58,34 @@ def __init__(self, token): target=calc_backlog_per_instance, args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) ).start() + threading.Thread( + target=send_videos_from_queue2, + args=(worker_to_bot_queue, config.get('bucket_name')) + ).start() def _message_handler(self, update, context): try: chat_id = str(update.effective_message.chat_id) - response = workers_queue.send_message( - MessageBody=update.message.text, - MessageAttributes={ - 'chat_id': {'StringValue': chat_id, 'DataType': 'String'} - } - ) - logger.info(f'msg {response.get("MessageId")} has been sent to queue') - self.send_text(update, f'Your message is being processed...', chat_id=chat_id) - if update.message.text.startswith('/myvideos'): response = table.query(KeyConditionExpression=Key('chatId').eq(chat_id)) for key, value in response.items(): - array_length = len(value) - for i in range(array_length): - temp_dict = value[i] - video_url = temp_dict['url'] - video = search_youtube_video(None, video_url) - self.send_text(update, f'Video title: {video["title"]}, Video Link: {video["webpage_url"]}', chat_id=chat_id) + if isinstance(value, list): + array_length = len(value) + for i in range(array_length): + temp_dict = value[i] + video_url = temp_dict['url'] + video = search_youtube_video(None, video_url) + self.send_text(update, f'Video Name: {video["title"]}, Video Link: {video["webpage_url"]}', chat_id=chat_id) + logger.info(f'sent videos information to client, chat_id: {chat_id}') else: + response = workers_queue.send_message( + MessageBody=update.message.text, + MessageAttributes={ + 'chat_id': {'StringValue': chat_id, 'DataType': 'String'} + } + ) + logger.info(f'msg {response.get("MessageId")} has been sent to queue') + self.send_text(update, f'Your message is being processed...', chat_id=chat_id) for video in search_youtube_video(update.message.text, None): item = { 'chatId': chat_id, @@ -103,6 +109,7 @@ def _message_handler(self, update, context): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) workers_queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) + worker_to_bot_queue = sqs.get_queue_by_name(QueueName=config.get('worker_to_bot_queue_name')) asg = boto3.client('autoscaling', region_name=config.get('aws_region')) dynamodb = boto3.resource('dynamodb', region_name=config.get('aws_region')) table = dynamodb.Table(config.get('table_name')) diff --git a/config.json b/config.json index ecc59525..0e0a30c2 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,7 @@ { "aws_region": "eu-central-1", "bot_to_worker_queue_name": "daniel-reuven-aws-ex1-polybot-queue", + "worker_to_bot_queue_name": "daniel-reuven-aws-ex1-polybot-queue2", "autoscaling_group_name": "daniel-reuven-polybot-asg", "cloudwatch_namespace": "cloudwatch-namespace", "bucket_name": "daniel-reuven-aws-ex1-polybot-bucket", diff --git a/utils.py b/utils.py index 194ac194..faf5f6b9 100644 --- a/utils.py +++ b/utils.py @@ -3,6 +3,7 @@ import os import boto3 import botocore +import requests from botocore.exceptions import ClientError from yt_dlp import YoutubeDL from loguru import logger @@ -25,13 +26,8 @@ def search_download_youtube_video(video_name, num_results, s3_bucket_name): # 1. get a list of video file names with download=false parameter videos = ydl.extract_info(f"ytsearch{num_results}:{video_name}", download=False)['entries'] for video in videos: - # if video['duration'] >= 900: - # return "Error, selected track/s are above predefined duration limit" - # if video['duration'] <= 0.1: - # return "Error, selected track/s are below predefined duration limit" localprefix = video['id'] + '.mp4' prefix = 'ytdlAppData/' + video['id'] + '.mp4' - # print(s3_bucket_name) # check aws s3 bucket for file, then locally and act accordingly,prefix != ydl.prepare_filename(video) if not (check_s3_file(prefix, s3_bucket_name)): if not (os.path.isfile(ydl.prepare_filename(video))): @@ -97,6 +93,42 @@ def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name, aws_ time.sleep(60) +def send_videos_from_queue2(sqs_queue_client2, bucket_name): + i = 0 + while True: + i += 1 + msgs_in_videos_queue = int(sqs_queue_client2.attributes.get('ApproximateNumberOfMessages')) + logger.info(f'msgs_in_videos_queue: {msgs_in_videos_queue}') + if msgs_in_videos_queue > 0: + logger.info(f'Attempting to send video to user via chat') + try: + messages = sqs_queue_client2.receive_messages( + MessageAttributeNames=['All'], + MaxNumberOfMessages=1, + WaitTimeSeconds=10 + ) + for msg in messages: + logger.info(f'processing message {msg}') + video_filename = msg.body + chat_id = msg.message_attributes.get('chat_id').get('StringValue') + video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) + send_message(chat_id, f'The following download link will be available for the next few minutes: {video_presigned_url}') + # delete message from the queue after it was handled + response = sqs_queue_client2.delete_messages(Entries=[{ + 'Id': msg.message_id, + 'ReceiptHandle': msg.receipt_handle + }]) + if 'Successful' in response: + logger.info(f'msg {msg} has been handled successfully') + logger.info(f'file has been downloaded') + except botocore.exceptions.ClientError as err: + logger.exception(f"Couldn't receive messages {err}") + if i == 6: + logger.info(f'Process is running, checking queue every 10 seconds, this msg repeats every 60 seconds') + i = 0 + sleep(10) + + def check_s3_file(key_filename, s3_bucket_name): s3_prefix = 'ytdlAppData/' + key_filename s3 = boto3.resource('s3') @@ -142,3 +174,32 @@ def download_file(key_filename, bucket, object_name=None): logger.error(e) return False return True + + +def generate_presigned_url(key_filename, bucket, object_name=None): + s3_prefix = 'ytdlAppData/' + key_filename + # Upload the file + s3_client = boto3.client('s3') + # If S3 object_name was not specified, use key_filename + if object_name is None: + object_name = s3_prefix + try: + response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': s3_prefix}, ExpiresIn=600) + except ClientError as e: + logger.error(e) + return False + return response + + +def send_message(chat_id, text): + with open('.telegramToken') as f: + _token = f.read() + url = f"https://api.telegram.org/bot{_token}/sendMessage" + params = { + "chat_id": chat_id, + "text": text, + } + resp = requests.get(url, params=params) + + # Throw an exception if Telegram API fails + resp.raise_for_status() diff --git a/worker.py b/worker.py index 51e9ac5e..50e76b74 100644 --- a/worker.py +++ b/worker.py @@ -8,7 +8,9 @@ def process_msg(msg): - search_download_youtube_video(msg, 1, s3_bucket_name) + video_filename = search_download_youtube_video(msg, 1, s3_bucket_name) + print(f'the video file name is : {video_filename}') + return video_filename def main(): @@ -21,8 +23,14 @@ def main(): ) for msg in messages: logger.info(f'processing message {msg}') - process_msg(msg.body) - + video_filename = process_msg(msg.body) + chat_id = msg.message_attributes.get('chat_id').get('StringValue') + response2 = worker_to_bot_queue.send_message( + MessageBody=video_filename[0], + MessageAttributes={'chat_id': {'StringValue': chat_id, 'DataType': 'String'} + } + ) + logger.info(f'msg {response2.get("MessageId")} has been sent to queue 2') # delete message from the queue after it was handled response = queue.delete_messages(Entries=[{ 'Id': msg.message_id, @@ -30,10 +38,9 @@ def main(): }]) if 'Successful' in response: logger.info(f'msg {msg} has been handled successfully') - except botocore.exceptions.ClientError as err: logger.exception(f"Couldn't receive messages {err}") - time.sleep(10) + time.sleep(10) if __name__ == '__main__': @@ -42,8 +49,8 @@ def main(): sqs = boto3.resource('sqs', region_name=config.get('aws_region')) queue = sqs.get_queue_by_name(QueueName=config.get('bot_to_worker_queue_name')) + worker_to_bot_queue = sqs.get_queue_by_name(QueueName=config.get('worker_to_bot_queue_name')) s3_bucket_name = config.get('bucket_name') - # print(s3_bucket_name) cwd = os.getcwd() path = f"{cwd}/ytdlAppData" # Check whether the specified path exists or not From c500e56c4cd8455b7a99d6a5ec1643080c588328 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sun, 31 Jul 2022 23:27:37 +0300 Subject: [PATCH 15/32] update, working code with presigned urls --- bot.py | 8 ++++---- requirements.txt | 3 ++- utils.py | 25 ++++++++++++------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/bot.py b/bot.py index 157f84c7..b0eb45a3 100644 --- a/bot.py +++ b/bot.py @@ -54,10 +54,10 @@ def _message_handler(self, update, context): class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) - threading.Thread( - target=calc_backlog_per_instance, - args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) - ).start() + # threading.Thread( + # target=calc_backlog_per_instance, + # args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) + # ).start() threading.Thread( target=send_videos_from_queue2, args=(worker_to_bot_queue, config.get('bucket_name')) diff --git a/requirements.txt b/requirements.txt index c9358c3b..a26e8fbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ python-telegram-bot>=13.11 yt-dlp>=2022.6.29 loguru~=0.6.0 botocore~=1.27.13 -boto3~=1.24.13 \ No newline at end of file +boto3~=1.24.13 +requests~=2.28.1 \ No newline at end of file diff --git a/utils.py b/utils.py index faf5f6b9..2ab50814 100644 --- a/utils.py +++ b/utils.py @@ -97,16 +97,15 @@ def send_videos_from_queue2(sqs_queue_client2, bucket_name): i = 0 while True: i += 1 - msgs_in_videos_queue = int(sqs_queue_client2.attributes.get('ApproximateNumberOfMessages')) - logger.info(f'msgs_in_videos_queue: {msgs_in_videos_queue}') - if msgs_in_videos_queue > 0: - logger.info(f'Attempting to send video to user via chat') - try: - messages = sqs_queue_client2.receive_messages( - MessageAttributeNames=['All'], - MaxNumberOfMessages=1, - WaitTimeSeconds=10 - ) + try: + messages = sqs_queue_client2.receive_messages( + MessageAttributeNames=['All'], + MaxNumberOfMessages=10, + WaitTimeSeconds=5 + ) + logger.info(f'msgs in videos queue: {len(messages)}') + if messages: + logger.info(f'Attempting to send video to user via chat') for msg in messages: logger.info(f'processing message {msg}') video_filename = msg.body @@ -121,8 +120,8 @@ def send_videos_from_queue2(sqs_queue_client2, bucket_name): if 'Successful' in response: logger.info(f'msg {msg} has been handled successfully') logger.info(f'file has been downloaded') - except botocore.exceptions.ClientError as err: - logger.exception(f"Couldn't receive messages {err}") + except botocore.exceptions.ClientError as err: + logger.exception(f"Couldn't receive messages {err}") if i == 6: logger.info(f'Process is running, checking queue every 10 seconds, this msg repeats every 60 seconds') i = 0 @@ -184,7 +183,7 @@ def generate_presigned_url(key_filename, bucket, object_name=None): if object_name is None: object_name = s3_prefix try: - response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': s3_prefix}, ExpiresIn=600) + response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': s3_prefix}, ExpiresIn=1800) except ClientError as e: logger.error(e) return False From 29c6af4420a719d6f264d06f156433b41d10a51f Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 17:37:09 +0300 Subject: [PATCH 16/32] update, working code with presigned urls --- utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils.py b/utils.py index 2ab50814..758d030d 100644 --- a/utils.py +++ b/utils.py @@ -199,6 +199,5 @@ def send_message(chat_id, text): "text": text, } resp = requests.get(url, params=params) - # Throw an exception if Telegram API fails resp.raise_for_status() From 7b9a86dcebdc55d68c25d836af80f22b63dbdc5d Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 17:49:51 +0300 Subject: [PATCH 17/32] update, working code with presigned urls --- worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/worker.py b/worker.py index 50e76b74..59c4abdf 100644 --- a/worker.py +++ b/worker.py @@ -40,6 +40,7 @@ def main(): logger.info(f'msg {msg} has been handled successfully') except botocore.exceptions.ClientError as err: logger.exception(f"Couldn't receive messages {err}") + logger.info(f'Waiting for new msgs') time.sleep(10) From 95dc9c4f96178ba27fa0b5de24457aa7ed25cd1f Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:12:12 +0300 Subject: [PATCH 18/32] update, working code with presigned urls --- utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 758d030d..56a9e04c 100644 --- a/utils.py +++ b/utils.py @@ -178,7 +178,8 @@ def download_file(key_filename, bucket, object_name=None): def generate_presigned_url(key_filename, bucket, object_name=None): s3_prefix = 'ytdlAppData/' + key_filename # Upload the file - s3_client = boto3.client('s3') + s3_client = boto3.client("s3", signature_version='s3v4') + # If S3 object_name was not specified, use key_filename if object_name is None: object_name = s3_prefix From bd2b0a587376a735b1b2f25c2f3d3e63556e1ecc Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:37:24 +0300 Subject: [PATCH 19/32] update, working code with presigned urls --- .gitignore | 1 + utils.py | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f37f683c..064d551b 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,4 @@ crashlytics-build.properties fabric.properties /asg-config.json +/aws-cred-config.json \ No newline at end of file diff --git a/utils.py b/utils.py index 56a9e04c..41cf042b 100644 --- a/utils.py +++ b/utils.py @@ -1,10 +1,11 @@ -# import logging +import json import time import os import boto3 import botocore import requests from botocore.exceptions import ClientError +from botocore.config import Config from yt_dlp import YoutubeDL from loguru import logger from time import sleep @@ -110,7 +111,9 @@ def send_videos_from_queue2(sqs_queue_client2, bucket_name): logger.info(f'processing message {msg}') video_filename = msg.body chat_id = msg.message_attributes.get('chat_id').get('StringValue') - video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) + # video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) + video_presigned_url = create_presigned_post(bucket_name, video_filename) + # def create_presigned_post(bucket_name, object_name, fields=None, conditions=None, expiration): send_message(chat_id, f'The following download link will be available for the next few minutes: {video_presigned_url}') # delete message from the queue after it was handled response = sqs_queue_client2.delete_messages(Entries=[{ @@ -191,6 +194,26 @@ def generate_presigned_url(key_filename, bucket, object_name=None): return response +def create_presigned_post(bucket_name, object_name): + with open('config.json') as f: + config = json.load(f) + with open('aws-cred-config.json') as f: + awscredconfig = json.load(f) + s3_prefix = 'ytdlAppData/' + object_name + # Generate a presigned S3 POST URL + s3_client = boto3.client("s3", config=Config(signature_version='s3v4'), region_name=config.get('aws_region'), aws_access_key_id=awscredconfig.get('aws_access_key_id'), + aws_secret_access_key=awscredconfig.get('aws_secret_access_key'), + aws_session_token="SESSION_TOKEN") + try: + response = s3_client.generate_presigned_post(Bucket=bucket_name, Key=s3_prefix, ExpiresIn=1800) + except ClientError as e: + return None + # The response contains the presigned URL and required fields + return response + + + + def send_message(chat_id, text): with open('.telegramToken') as f: _token = f.read() From bc580e0f015562a76cd22475e83cdfc41c092034 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:41:01 +0300 Subject: [PATCH 20/32] update, working code with presigned urls --- utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 41cf042b..9b862c58 100644 --- a/utils.py +++ b/utils.py @@ -113,7 +113,6 @@ def send_videos_from_queue2(sqs_queue_client2, bucket_name): chat_id = msg.message_attributes.get('chat_id').get('StringValue') # video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) video_presigned_url = create_presigned_post(bucket_name, video_filename) - # def create_presigned_post(bucket_name, object_name, fields=None, conditions=None, expiration): send_message(chat_id, f'The following download link will be available for the next few minutes: {video_presigned_url}') # delete message from the queue after it was handled response = sqs_queue_client2.delete_messages(Entries=[{ @@ -205,7 +204,8 @@ def create_presigned_post(bucket_name, object_name): aws_secret_access_key=awscredconfig.get('aws_secret_access_key'), aws_session_token="SESSION_TOKEN") try: - response = s3_client.generate_presigned_post(Bucket=bucket_name, Key=s3_prefix, ExpiresIn=1800) + # response = s3_client.generate_presigned_url(Bucket=bucket_name, Key=s3_prefix, ExpiresIn=1800) + response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket_name,'Key': s3_prefix}, ExpiresIn=1800) except ClientError as e: return None # The response contains the presigned URL and required fields From 8e0fa7cff30156e78edb990ca7e7b1da2b654019 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:47:11 +0300 Subject: [PATCH 21/32] update, working code with presigned urls --- utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/utils.py b/utils.py index 9b862c58..e1d77b7b 100644 --- a/utils.py +++ b/utils.py @@ -200,20 +200,18 @@ def create_presigned_post(bucket_name, object_name): awscredconfig = json.load(f) s3_prefix = 'ytdlAppData/' + object_name # Generate a presigned S3 POST URL - s3_client = boto3.client("s3", config=Config(signature_version='s3v4'), region_name=config.get('aws_region'), aws_access_key_id=awscredconfig.get('aws_access_key_id'), - aws_secret_access_key=awscredconfig.get('aws_secret_access_key'), - aws_session_token="SESSION_TOKEN") + # s3_client = boto3.client("s3", config=Config(signature_version='s3v4'), region_name=config.get('aws_region'), aws_access_key_id=awscredconfig.get('aws_access_key_id'), + # aws_secret_access_key=awscredconfig.get('aws_secret_access_key'), + # aws_session_token="SESSION_TOKEN") + s3_client = boto3.client("s3") try: - # response = s3_client.generate_presigned_url(Bucket=bucket_name, Key=s3_prefix, ExpiresIn=1800) - response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket_name,'Key': s3_prefix}, ExpiresIn=1800) + response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket_name, 'Key': s3_prefix}, ExpiresIn=1800) except ClientError as e: return None # The response contains the presigned URL and required fields return response - - def send_message(chat_id, text): with open('.telegramToken') as f: _token = f.read() From f80c5fa1aa14fac07c802a501610f8d5d1718cbd Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:47:31 +0300 Subject: [PATCH 22/32] update, testing code with presigned urls --- utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/utils.py b/utils.py index e1d77b7b..7402ce6c 100644 --- a/utils.py +++ b/utils.py @@ -196,13 +196,9 @@ def generate_presigned_url(key_filename, bucket, object_name=None): def create_presigned_post(bucket_name, object_name): with open('config.json') as f: config = json.load(f) - with open('aws-cred-config.json') as f: - awscredconfig = json.load(f) s3_prefix = 'ytdlAppData/' + object_name # Generate a presigned S3 POST URL - # s3_client = boto3.client("s3", config=Config(signature_version='s3v4'), region_name=config.get('aws_region'), aws_access_key_id=awscredconfig.get('aws_access_key_id'), - # aws_secret_access_key=awscredconfig.get('aws_secret_access_key'), - # aws_session_token="SESSION_TOKEN") + s3_client = boto3.client("s3") try: response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket_name, 'Key': s3_prefix}, ExpiresIn=1800) From bee672b202f1ec909ec5a1ad2f5700f46c3a2028 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 18:53:12 +0300 Subject: [PATCH 23/32] update, testing code with presigned urls 2 --- .gitignore | 3 +-- utils.py | 20 ++------------------ 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 064d551b..bc2ec3d0 100644 --- a/.gitignore +++ b/.gitignore @@ -157,5 +157,4 @@ crashlytics.properties crashlytics-build.properties fabric.properties -/asg-config.json -/aws-cred-config.json \ No newline at end of file +/asg-config.json \ No newline at end of file diff --git a/utils.py b/utils.py index 7402ce6c..1f060bc9 100644 --- a/utils.py +++ b/utils.py @@ -111,8 +111,7 @@ def send_videos_from_queue2(sqs_queue_client2, bucket_name): logger.info(f'processing message {msg}') video_filename = msg.body chat_id = msg.message_attributes.get('chat_id').get('StringValue') - # video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) - video_presigned_url = create_presigned_post(bucket_name, video_filename) + video_presigned_url = generate_presigned_url(video_filename, bucket_name, None) send_message(chat_id, f'The following download link will be available for the next few minutes: {video_presigned_url}') # delete message from the queue after it was handled response = sqs_queue_client2.delete_messages(Entries=[{ @@ -180,7 +179,7 @@ def download_file(key_filename, bucket, object_name=None): def generate_presigned_url(key_filename, bucket, object_name=None): s3_prefix = 'ytdlAppData/' + key_filename # Upload the file - s3_client = boto3.client("s3", signature_version='s3v4') + s3_client = boto3.client("s3", config=Config(signature_version='s3v4')) # If S3 object_name was not specified, use key_filename if object_name is None: @@ -193,21 +192,6 @@ def generate_presigned_url(key_filename, bucket, object_name=None): return response -def create_presigned_post(bucket_name, object_name): - with open('config.json') as f: - config = json.load(f) - s3_prefix = 'ytdlAppData/' + object_name - # Generate a presigned S3 POST URL - - s3_client = boto3.client("s3") - try: - response = s3_client.generate_presigned_url('get_object', Params={'Bucket': bucket_name, 'Key': s3_prefix}, ExpiresIn=1800) - except ClientError as e: - return None - # The response contains the presigned URL and required fields - return response - - def send_message(chat_id, text): with open('.telegramToken') as f: _token = f.read() From aef67ace8f56df5d4df6da199ca43d3b181fb90c Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 19:00:02 +0300 Subject: [PATCH 24/32] update, testing code with presigned urls 3 --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 1f060bc9..ce9d010d 100644 --- a/utils.py +++ b/utils.py @@ -179,7 +179,7 @@ def download_file(key_filename, bucket, object_name=None): def generate_presigned_url(key_filename, bucket, object_name=None): s3_prefix = 'ytdlAppData/' + key_filename # Upload the file - s3_client = boto3.client("s3", config=Config(signature_version='s3v4')) + s3_client = boto3.client("s3", 'eu-central-1', config=Config(signature_version='s3v4')) # If S3 object_name was not specified, use key_filename if object_name is None: From f7bcae7499f1e0fe5d0d87544a7e3c854c8e8763 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Wed, 3 Aug 2022 19:04:18 +0300 Subject: [PATCH 25/32] update, finished code --- bot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot.py b/bot.py index b0eb45a3..157f84c7 100644 --- a/bot.py +++ b/bot.py @@ -54,10 +54,10 @@ def _message_handler(self, update, context): class YoutubeObjectDetectBot(Bot): def __init__(self, token): super().__init__(token) - # threading.Thread( - # target=calc_backlog_per_instance, - # args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) - # ).start() + threading.Thread( + target=calc_backlog_per_instance, + args=(workers_queue, asg, config.get("autoscaling_group_name"), config.get('aws_region')) + ).start() threading.Thread( target=send_videos_from_queue2, args=(worker_to_bot_queue, config.get('bucket_name')) From b33c4c1bac739dddfdc30791b92893c20c6086fc Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 17:17:41 +0200 Subject: [PATCH 26/32] testing --- PR.Jenkinsfile | 16 ++++++++++++++++ tests/test_autoscaling_metric.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 PR.Jenkinsfile create mode 100644 tests/test_autoscaling_metric.py diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile new file mode 100644 index 00000000..ea455cb0 --- /dev/null +++ b/PR.Jenkinsfile @@ -0,0 +1,16 @@ +pipeline { + agent any + + stages { + stage('Unittest') { + steps { + echo "testing" + } + } + stage('Functional test') { + steps { + echo "testing" + } + } + } +} \ No newline at end of file diff --git a/tests/test_autoscaling_metric.py b/tests/test_autoscaling_metric.py new file mode 100644 index 00000000..ca57848d --- /dev/null +++ b/tests/test_autoscaling_metric.py @@ -0,0 +1,22 @@ +import unittest2 as unittest +from unittest.mock import Mock +from utils import calc_backlog_per_instance + + +class TestBacklogPerInstanceMetric(unittest.TestCase): + def setUp(self): + self.sqs_queue_client = Mock() + self.asg_client = Mock() + + def test_no_worker_full_queue(self): + self.sqs_queue_client.attributes = { + 'ApproximateNumberOfMessages': '100' + } + + self.asg_client.describe_auto_scaling_groups = Mock(return_value={ + 'AutoScalingGroups': [{ + 'DesiredCapacity': 0 + }] + }) + + self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None), 99) From 65f1dac3f344f1f25434d782e33e1f86bad17dbe Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 22:16:30 +0200 Subject: [PATCH 27/32] uni testing --- PR.Jenkinsfile | 10 +++++++++- requirements.txt | 3 ++- tests/test_autoscaling_metric.py | 30 +++++++++++++++++++++++++++++- utils.py | 25 +++++++++++++------------ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile index ea455cb0..44a13c6b 100644 --- a/PR.Jenkinsfile +++ b/PR.Jenkinsfile @@ -4,7 +4,15 @@ pipeline { stages { stage('Unittest') { steps { - echo "testing" + sh ''' + pip install -r requirements.txt + python -m pytest --junitxml results.xml tests + ''' + } + post { + always { + junit allowEmptyResults: true, testResults: 'results.xml' + } } } stage('Functional test') { diff --git a/requirements.txt b/requirements.txt index a26e8fbd..38575cb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ yt-dlp>=2022.6.29 loguru~=0.6.0 botocore~=1.27.13 boto3~=1.24.13 -requests~=2.28.1 \ No newline at end of file +requests~=2.28.1 +unittest2~=1.1.0 \ No newline at end of file diff --git a/tests/test_autoscaling_metric.py b/tests/test_autoscaling_metric.py index ca57848d..9b462a0b 100644 --- a/tests/test_autoscaling_metric.py +++ b/tests/test_autoscaling_metric.py @@ -2,6 +2,8 @@ from unittest.mock import Mock from utils import calc_backlog_per_instance +# run by `PYTHONPATH=. python3 -m pytest --junitxml results.xml tests` + class TestBacklogPerInstanceMetric(unittest.TestCase): def setUp(self): @@ -19,4 +21,30 @@ def test_no_worker_full_queue(self): }] }) - self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None), 99) + self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None, None), 99) + + def test_no_workers_empty_queue(self): + self.sqs_queue_client.attributes = { + 'ApproximateNumberOfMessages': '0' + } + + self.asg_client.describe_auto_scaling_groups = Mock(return_value={ + 'AutoScalingGroups': [{ + 'DesiredCapacity': 0 + }] + }) + + self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None, None), 0) + + def test_2_workers_100_msgs_in_queue(self): + self.sqs_queue_client.attributes = { + 'ApproximateNumberOfMessages': '100' + } + + self.asg_client.describe_auto_scaling_groups = Mock(return_value={ + 'AutoScalingGroups': [{ + 'DesiredCapacity': 2 + }] + }) + + self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None, None), 50) diff --git a/utils.py b/utils.py index ce9d010d..4b481f70 100644 --- a/utils.py +++ b/utils.py @@ -79,19 +79,20 @@ def calc_backlog_per_instance(sqs_queue_client, asg_client, asg_group_name, aws_ backlog_per_instance = msgs_in_queue / asg_size logger.info(f'backlog per instance: {backlog_per_instance}') # Create CloudWatch client - cloudwatch = boto3.client('cloudwatch', aws_region) + # cloudwatch = boto3.client('cloudwatch', aws_region) # Put custom metrics - cloudwatch.put_metric_data( - Namespace='daniel-reuven-monitor-polybot-asg', - MetricData=[ - { - 'MetricName': 'backlog_per_instance', - 'Value': backlog_per_instance, - 'Unit': 'Count' - }, - ] - ) - time.sleep(60) + # cloudwatch.put_metric_data( + # Namespace='daniel-reuven-monitor-polybot-asg', + # MetricData=[ + # { + # 'MetricName': 'backlog_per_instance', + # 'Value': backlog_per_instance, + # 'Unit': 'Count' + # }, + # ] + #) + # time.sleep(60) + return backlog_per_instance def send_videos_from_queue2(sqs_queue_client2, bucket_name): From d30d6f9418cfb7dd692634cfcb7fa5b83cb9ff25 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 22:17:29 +0200 Subject: [PATCH 28/32] uni testing --- PR.Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile index 44a13c6b..76e308fa 100644 --- a/PR.Jenkinsfile +++ b/PR.Jenkinsfile @@ -5,7 +5,7 @@ pipeline { stage('Unittest') { steps { sh ''' - pip install -r requirements.txt + pip3 install -r requirements.txt python -m pytest --junitxml results.xml tests ''' } From b410a4381d10e40de994fa8b85705766e8d85649 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 22:18:58 +0200 Subject: [PATCH 29/32] uni testing 3 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38575cb7..aa3de062 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ loguru~=0.6.0 botocore~=1.27.13 boto3~=1.24.13 requests~=2.28.1 -unittest2~=1.1.0 \ No newline at end of file +unittest2~=1.1.0 +pytest \ No newline at end of file From bb6e6a9f300c54158872fc0e31c4cd710e7b8b1e Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 22:20:36 +0200 Subject: [PATCH 30/32] uni testing 3 --- PR.Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile index 76e308fa..d89b7c53 100644 --- a/PR.Jenkinsfile +++ b/PR.Jenkinsfile @@ -6,7 +6,7 @@ pipeline { steps { sh ''' pip3 install -r requirements.txt - python -m pytest --junitxml results.xml tests + python3 -m pytest --junitxml results.xml tests ''' } post { From b1b016771a561f584f6fe04785e7438507f512ff Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Sat, 12 Nov 2022 22:20:57 +0200 Subject: [PATCH 31/32] uni testing 4 --- PR.Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile index d89b7c53..fdd1d604 100644 --- a/PR.Jenkinsfile +++ b/PR.Jenkinsfile @@ -21,4 +21,4 @@ pipeline { } } } -} \ No newline at end of file +} From 9b6d24595d4bda927854a87378f82b08af664a08 Mon Sep 17 00:00:00 2001 From: DanielAdmin Date: Tue, 29 Nov 2022 16:49:39 +0200 Subject: [PATCH 32/32] add pylint --- Dockerfile | 11 +++++------ PR.Jenkinsfile | 15 +++++++++++++++ requirements.txt | 3 ++- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6f6d50e4..8dad5a09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ -FROM python:3.8.12-slim-buster - -# YOUR COMMANDS HERE -# .... -# .... - +FROM python:3.8-slim-bullseye +WORKDIR /botapp +LABEL app=bot +COPY . . +RUN pip install -r requirements.txt CMD ["python3", "bot.py"] \ No newline at end of file diff --git a/PR.Jenkinsfile b/PR.Jenkinsfile index fdd1d604..eba9cb91 100644 --- a/PR.Jenkinsfile +++ b/PR.Jenkinsfile @@ -20,5 +20,20 @@ pipeline { echo "testing" } } + stage('Static code linting') { + steps { + sh 'python3 -m pylint -f parseable --reports=no *.py > pylint.log' + } + post { + always { + sh 'cat pylint.log' + recordIssues ( + enabledForFailure: true, + aggregatingResults: true, + tools: [pyLint(name: 'Pylint', pattern: '**/pylint.log')] + ) + } + } + } } } diff --git a/requirements.txt b/requirements.txt index aa3de062..442653ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ botocore~=1.27.13 boto3~=1.24.13 requests~=2.28.1 unittest2~=1.1.0 -pytest \ No newline at end of file +pytest +pylint \ No newline at end of file