From 542b2615c07360e47f95bd112a3aabc88f5a6a3d Mon Sep 17 00:00:00 2001 From: feng <471899214@qq.com> Date: Fri, 25 Oct 2024 18:56:04 +0800 Subject: [PATCH 1/6] migrate to r2 instead of hf --- full_automation.py | 41 +++++++++---------------- requirements.txt | 1 + utils/cloudflare_utils.py | 63 +++++++++++++++++++++++++++++++++++++++ utils/flock_api.py | 8 ++--- 4 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 utils/cloudflare_utils.py diff --git a/full_automation.py b/full_automation.py index 2aab6a1..e82ad54 100644 --- a/full_automation.py +++ b/full_automation.py @@ -5,12 +5,12 @@ import requests import yaml from loguru import logger -from huggingface_hub import HfApi from demo import LoraTrainingArguments, train_lora from utils.constants import model2base_model, model2size -from utils.flock_api import get_task, submit_task +from utils.flock_api import get_task, get_address from utils.gpu_utils import get_gpu_type +from utils.cloudflare_utils import CloudStorage HF_USERNAME = os.environ["HF_USERNAME"] @@ -59,34 +59,21 @@ gpu_type = get_gpu_type() try: - logger.info("Start to push the lora weight to the hub...") - api = HfApi(token=os.environ["HF_TOKEN"]) - repo_name = f"{HF_USERNAME}/task-{task_id}-{model_id.replace('/', '-')}" - # check whether the repo exists - try: - api.create_repo( - repo_name, - exist_ok=False, - repo_type="model", - ) - except Exception as e: - logger.info( - f"Repo {repo_name} already exists. Will commit the new version." - ) + logger.info("Start to push the lora weight to the cloudflare R2...") - commit_message = api.upload_folder( - folder_path="outputs", - repo_id=repo_name, - repo_type="model", + upload_data = get_address( + task_id, model2base_model[model_id], gpu_type ) - # get commit hash - commit_hash = commit_message.oid - logger.info(f"Commit hash: {commit_hash}") - logger.info(f"Repo name: {repo_name}") - # submit - submit_task( - task_id, repo_name, model2base_model[model_id], gpu_type, commit_hash + cf_storage = CloudStorage( + access_key=upload_data["data"]["access_key"], + secret_key=upload_data["data"]["secret_key"], + endpoint_url=upload_data["data"]["endpoint_url"], + session_token=upload_data["data"]["session_token"], + bucket=upload_data["data"]["bucket"] ) + cf_storage.initialize() + cf_storage.upload_folder(local_folder="outputs", + cloud_path=upload_data["data"]["folder_name"]) logger.info("Task submitted successfully") except Exception as e: logger.error(f"Error: {e}") diff --git a/requirements.txt b/requirements.txt index 9b73f7c..4370080 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ loguru trl>=0.9.3,<=0.9.6 bitsandbytes pyyaml +boto3 diff --git a/utils/cloudflare_utils.py b/utils/cloudflare_utils.py new file mode 100644 index 0000000..1721e2c --- /dev/null +++ b/utils/cloudflare_utils.py @@ -0,0 +1,63 @@ +import os +import boto3 +import botocore +import threading +from tqdm import tqdm +from loguru import logger + + +class ProgressPercentage: + def __init__(self, filename): + self.filename = filename + self.size = float(os.path.getsize(filename)) + self._seen_so_far = 0 + self._lock = threading.Lock() + + def __call__(self, bytes_amount): + with self._lock: + self._seen_so_far += bytes_amount + percentage = (self._seen_so_far / self.size) * 100 + logger.info("\r%s %s / %s (%.2f%%)" % ( + self.filename, self._seen_so_far, self.size, + percentage), end="") + + +class CloudStorage: + def __init__(self, access_key=None, secret_key=None, endpoint_url=None, bucket=None, session_token=None): + self.access_key = access_key + self.secret_key = secret_key + self.endpoint_url = endpoint_url + self.bucket = bucket + self.client = None + self.session_token = session_token + + def initialize(self): + if self.access_key is None or self.secret_key is None or self.endpoint_url is None: + logger.error("Please provide access_key, secret_key, session_token and endpoint_url") + raise + self.client = boto3.client('s3', + endpoint_url=self.endpoint_url, + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token + ) + return self + + def upload_folder(self, local_folder, cloud_path, bucket=None): + if bucket is None and self.bucket is None: + logger.error("Please provide bucket name") + return + if bucket is None: + bucket = self.bucket + stream = tqdm(os.walk(local_folder)) + for root, dirs, files in stream: + for file in files: + localFilePath = os.path.join(root, file) + relativePath = os.path.relpath(localFilePath, local_folder) + cloudPath = os.path.join(cloud_path, relativePath) + cloudPath = cloudPath.replace('\\', '/') + try: + self.client.upload_file(localFilePath, bucket, cloudPath, + Callback=ProgressPercentage(localFilePath)) + except botocore.exceptions.ClientError as e: + logger.error(e) diff --git a/utils/flock_api.py b/utils/flock_api.py index f62fc67..72d4094 100644 --- a/utils/flock_api.py +++ b/utils/flock_api.py @@ -14,17 +14,15 @@ def get_task(task_id: int): return response.json() -def submit_task( - task_id: int, hg_repo_id: str, base_model: str, gpu_type: str, revision: str +def get_address( + task_id: int, base_model: str, gpu_type: str ): payload = json.dumps( { "task_id": task_id, "data": { - "hg_repo_id": hg_repo_id, "base_model": base_model, "gpu_type": gpu_type, - "revision": revision, }, } ) @@ -34,7 +32,7 @@ def submit_task( } response = requests.request( "POST", - f"{FED_LEDGER_BASE_URL}/tasks/submit-result", + f"{FED_LEDGER_BASE_URL}/tasks/get_storage_credentials", headers=headers, data=payload, ) From 773a26c3626742bde574748d99a80b4f2cc3313f Mon Sep 17 00:00:00 2001 From: feng <471899214@qq.com> Date: Wed, 30 Oct 2024 17:06:53 +0800 Subject: [PATCH 2/6] migrate to r2 instead of hf --- full_automation.py | 12 ++++++------ utils/flock_api.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/full_automation.py b/full_automation.py index e82ad54..f9c51f4 100644 --- a/full_automation.py +++ b/full_automation.py @@ -8,12 +8,10 @@ from demo import LoraTrainingArguments, train_lora from utils.constants import model2base_model, model2size -from utils.flock_api import get_task, get_address +from utils.flock_api import get_task, submit_task, get_address from utils.gpu_utils import get_gpu_type from utils.cloudflare_utils import CloudStorage -HF_USERNAME = os.environ["HF_USERNAME"] - if __name__ == "__main__": task_id = os.environ["TASK_ID"] # load trainin args @@ -61,9 +59,7 @@ try: logger.info("Start to push the lora weight to the cloudflare R2...") - upload_data = get_address( - task_id, model2base_model[model_id], gpu_type - ) + upload_data = get_address(task_id) cf_storage = CloudStorage( access_key=upload_data["data"]["access_key"], secret_key=upload_data["data"]["secret_key"], @@ -74,6 +70,10 @@ cf_storage.initialize() cf_storage.upload_folder(local_folder="outputs", cloud_path=upload_data["data"]["folder_name"]) + submit_task( + task_id, model2base_model[model_id], gpu_type, + upload_data["data"]["bucket"], upload_data["data"]["folder_name"] + ) logger.info("Task submitted successfully") except Exception as e: logger.error(f"Error: {e}") diff --git a/utils/flock_api.py b/utils/flock_api.py index 72d4094..e2fdfbf 100644 --- a/utils/flock_api.py +++ b/utils/flock_api.py @@ -14,15 +14,18 @@ def get_task(task_id: int): return response.json() -def get_address( - task_id: int, base_model: str, gpu_type: str +def submit_task( + task_id: int, base_model: str, gpu_type: str, bucket: str, folder_name: str ): payload = json.dumps( { "task_id": task_id, "data": { + "base_model": base_model, "gpu_type": gpu_type, + "bucket": bucket, + "folder_name": folder_name, }, } ) @@ -30,6 +33,29 @@ def get_address( "flock-api-key": FLOCK_API_KEY, "Content-Type": "application/json", } + response = requests.request( + "POST", + f"{FED_LEDGER_BASE_URL}/tasks/submit-result", + headers=headers, + data=payload, + ) + if response.status_code != 200: + raise Exception(f"Failed to submit task: {response.text}") + return response.json() + + +def get_address( + task_id: int +): + payload = json.dumps( + { + "task_id": task_id, + } + ) + headers = { + "flock-api-key": FLOCK_API_KEY, + "Content-Type": "application/json", + } response = requests.request( "POST", f"{FED_LEDGER_BASE_URL}/tasks/get_storage_credentials", From f4f0ea6b3fc4e2ef77a476d82728d2fc572458cd Mon Sep 17 00:00:00 2001 From: feng <471899214@qq.com> Date: Wed, 30 Oct 2024 20:53:34 +0800 Subject: [PATCH 3/6] migrate to r2 instead of hf --- demo.py | 1 - full_automation.py | 1 - merge.py | 1 - 3 files changed, 3 deletions(-) diff --git a/demo.py b/demo.py index ade7c37..5e87ab3 100644 --- a/demo.py +++ b/demo.py @@ -7,7 +7,6 @@ from trl import SFTTrainer, SFTConfig from dataset import SFTDataCollator, SFTDataset -from merge import merge_lora_to_base_model from utils.constants import model2template diff --git a/full_automation.py b/full_automation.py index f9c51f4..5044661 100644 --- a/full_automation.py +++ b/full_automation.py @@ -1,6 +1,5 @@ import json import os -import time import requests import yaml diff --git a/merge.py b/merge.py index 80cb6ab..2580bb3 100644 --- a/merge.py +++ b/merge.py @@ -1,4 +1,3 @@ -import json import torch from peft import PeftModel from transformers import AutoModelForCausalLM, AutoTokenizer From 822ec0606436b5901ad04125b7d74fbd2c193f66 Mon Sep 17 00:00:00 2001 From: feng <471899214@qq.com> Date: Mon, 4 Nov 2024 16:54:45 +0800 Subject: [PATCH 4/6] migrate to r2 instead of hf --- .pre-commit-config.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fc79e09 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.6.4 + hooks: + # Run the linter. + - id: ruff + # Run the formatter. + - id: ruff-format From c9c3b093b34a0215def1278d1f9f46f141e99308 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:54:54 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Dockerfile | 2 +- README.md | 2 +- demo_data.jsonl | 2 +- full_automation.py | 14 +++++++---- utils/cloudflare_utils.py | 50 +++++++++++++++++++++++++++------------ utils/flock_api.py | 7 ++---- 6 files changed, 49 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 935ab34..3001049 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,4 @@ RUN pip3 install --no-cache-dir -r requirements.txt # Copy the rest of the application code COPY . . -CMD ["python", "full_automation.py"] \ No newline at end of file +CMD ["python", "full_automation.py"] diff --git a/README.md b/README.md index 57dded9..0d892b8 100644 --- a/README.md +++ b/README.md @@ -87,4 +87,4 @@ curl --location 'https://fed-ledger-prod.flock.io/api/v1/tasks/submit-result' \ - `hg_repo_id`: The Hugging Face repository where the model is stored, typically in the format `username/repository-name`. - `base_model`: The base model used for training. A list of all supported models can be found [here](https://github.com/FLock-io/llm-loss-validator/blob/main/src/core/constant.py). - `gpu_type`: The type of GPU used for training the model. -- `revision`: The commit hash from the Hugging Face repository. This uniquely identifies the version of the model that was trained and submitted, allowing for precise tracking of changes and updates. \ No newline at end of file +- `revision`: The commit hash from the Hugging Face repository. This uniquely identifies the version of the model that was trained and submitted, allowing for precise tracking of changes and updates. diff --git a/demo_data.jsonl b/demo_data.jsonl index e6940c0..4f16bd7 100644 --- a/demo_data.jsonl +++ b/demo_data.jsonl @@ -5,4 +5,4 @@ {"system": "You are a helpful assistant.", "conversations": [{"role": "user", "content": "What tips do you have for balancing work and home life?"}, {"role": "assistant", "content": "Here are a few tips for balancing work and home life:\n\n1. Set clear boundaries: Establish clear boundaries between work and home life. Try to stick to a regular work schedule and avoid working during your personal time.\n\n2. Prioritize your tasks: Prioritize your tasks to make sure you are accomplishing the most important things first. This can help you avoid wasting time on less important tasks and free up more time for your personal life.\n\n3. Take breaks: Taking regular breaks can help you stay focused and avoid burnout. Use your breaks to relax, exercise, or spend time with loved ones.\n\n4. Use technology to your advantage: Use technology to help you manage your work and personal life. For example, use a calendar app to schedule your work and personal commitments, or use a task management app to keep track of your to-do list.\n\n5. Learn to say no: Saying no to work commitments that don't align with your priorities can free up more time for your personal life.\n\nRemember that achieving a perfect balance between work and home life is not always possible. But by implementing these tips, you can create a healthy balance that works for you."}, {"role": "user", "content": "How can I develop better time management skills?"}, {"role": "assistant", "content": "Developing better time management skills can be challenging, but it is important for achieving your personal and professional goals. Here are some tips to help you improve your time management skills:\n\n1. Track your time: Start by tracking your time for a week or two to get a sense of how you are currently spending your time. This will help you identify any time-wasting activities and prioritize your tasks.\n\n2. Set goals: Set clear and specific goals for what you want to achieve in your personal and professional life. This will help you prioritize your tasks and stay focused on what really matters.\n\n3. Prioritize your tasks: Identify the most important tasks and prioritize them based on their urgency and importance. Use a to-do list or a task management app to keep track of your tasks.\n\n4. Use time management tools: There are many time management tools available, such as calendars, timers, and task lists. Use these tools to help you stay organized and on track.\n\n5. Learn to say no: Saying no to non-essential tasks or commitments can help you free up more time for the things that matter most.\n\n6. Take breaks: Taking regular breaks can help you stay focused and avoid burnout. Use your breaks to relax, exercise, or spend time with loved ones.\n\nRemember that developing better time management skills takes time and practice. Be patient with yourself and stay committed to your goals."}, {"role": "user", "content": "Can you recommend any particular time management tools that would be helpful?"}, {"role": "assistant", "content": "Sure! Here are some time management tools that you may find helpful:\n\n1. Calendar apps: Calendar apps like Google Calendar and Apple Calendar can help you schedule your appointments and tasks, set reminders, and keep track of your schedule.\n\n2. Task management apps: Task management apps like Trello, Asana, and Todoist can help you create to-do lists, set deadlines for your tasks, and prioritize your work.\n\n3. Pomodoro timers: Pomodoro timers like Focus@Will and Tomato Timer can help you stay focused by breaking your work into small, manageable chunks of time.\n\n4. Time-tracking apps: Time-tracking apps like RescueTime and Toggl can help you monitor how much time you spend on different tasks and activities, and identify areas where you can improve your productivity.\n\n5. Note-taking apps: Note-taking apps like Evernote and OneNote can help you jot down ideas, take notes during meetings, and keep track of important information.\n\n6. Distraction-blocker apps: Distraction-blocker apps like Freedom and Cold Turkey can help you stay focused by blocking distracting websites and apps during set times of day.\n\nRemember, the key to effective time management is finding a tool or set of tools that work best for you, and then using them consistently to help you stay focused and productive."}, {"role": "user", "content": "How can I improve my work efficiency?"}, {"role": "assistant", "content": "Improving your work efficiency can help you be more productive, complete your tasks more quickly, and reduce your stress levels. Here are some tips to help you improve your work efficiency:\n\n1. Set clear goals: Set clear and specific goals for what you want to achieve and break them down into smaller, actionable steps.\n\n2. Eliminate distractions: Eliminate distractions as much as possible by turning off notifications, closing unnecessary tabs or apps, and finding a quiet space to work.\n\n3. Prioritize your tasks: Identify the most important tasks and prioritize them based on their urgency and importance.\n\n4. Use time management tools: Use tools like calendars, timers, and task lists to help you stay organized and on track.\n\n5. Automate tasks: Automate tasks like email management and scheduling appointments to free up more time for important work.\n\n6. Delegate tasks: Delegate tasks where possible to colleagues or employees to help you focus on your most important responsibilities.\n\n7. Take breaks: Taking regular breaks can help you stay focused and avoid burnout. Use your breaks to relax, exercise, or spend time with loved ones.\n\nRemember, improving your work efficiency takes time and practice. Be patient with yourself and stay committed to your goals."}]} {"system": "You are a helpful assistant.", "conversations": [{"role": "user", "content": "What best practices should I follow when designing a website?"}, {"role": "assistant", "content": "Here are some best practices to follow when designing a website:\n\n1. Keep it simple and easy to navigate: Make sure your website is easy to navigate and that visitors can find what they are looking for quickly.\n\n2. Use a responsive design: Ensure that your website is optimized for all devices and screen sizes, including mobile phones and tablets.\n\n3. Use high-quality images and graphics: Use high-quality images and graphics to make your website visually appealing.\n\n4. Use colors and fonts consistently: Choose colors and fonts that match your brand and use them consistently throughout your website.\n\n5. Make it easy to read: Use clear and concise language, and choose a font size and style that is easy to read.\n\n6. Use white space effectively: Use white space to make your website look less cluttered and easier to read.\n\n7. Ensure fast loading speeds: Optimize your website's loading speed to ensure that visitors don't have to wait long to access your content.\n\n8. Use SEO best practices: Optimize your website for search engines by using keywords, meta-descriptions, and other SEO best practices.\n\n9. Test your website: Regularly test your website to ensure that it is functioning properly and that any issues are addressed quickly.\n\nBy following these best practices, you can create a website that is user-friendly, visually appealing, and easy to navigate, and that effectively communicates your brand's message to your target audience."}, {"role": "user", "content": "How can I optimize my website's loading speed?"}, {"role": "assistant", "content": "Here are some ways to optimize your website's loading speed:\n\n1. Minimize HTTP requests: Minimize the number of HTTP requests by reducing the number of files on your website, combining files, and using CSS sprites.\n\n2. Optimize images: Optimize your images by compressing them, reducing their size, and using the right file format.\n\n3. Use a content delivery network (CDN): Use a CDN to reduce the load time of your website by serving content from the nearest server to the user.\n\n4. Enable browser caching: Enable browser caching to store static files on the user's computer, which reduces the time it takes to load the website on subsequent visits.\n\n5. Minimize plugins and scripts: Minimize the number of plugins and scripts used on your website to reduce their impact on loading speed.\n\n6. Reduce server response time: Optimize your server response time by using a fast hosting provider, reducing the number of requests, and using caching and content compression.\n\n7. Use a fast theme or template: Use a fast theme or template that is well-optimized for speed.\n\n8. Minimize the use of redirects: Minimize the use of redirects, as they can slow down the loading speed of your website.\n\nBy following these tips, you can optimize your website's loading speed and create a better user experience for your visitors."}, {"role": "user", "content": "How can I make sure my website is optimized for search engines?"}, {"role": "assistant", "content": "Here are some ways to optimize your website for search engines:\n\n1. Conduct keyword research: Conduct keyword research to identify the keywords and phrases that your target audience is using to search for your products or services.\n\n2. Use keywords in your content: Use your target keywords in your content, including in your titles, headings, meta descriptions, and body copy.\n\n3. Create high-quality content: Create high-quality, informative content that provides value to your target audience and encourages them to share it.\n\n4. Use meta descriptions and title tags: Use meta descriptions and title tags to provide a brief summary of your content and make it easier for search engines to understand what your website is about.\n\n5. Use header tags: Use header tags (H1, H2, H3, etc.) to organize your content and make it easier for search engines to understand the hierarchy of your content.\n\n6. Optimize images: Optimize your images by using descriptive filenames, alt tags, and captions that include your target keywords.\n\n7. Build high-quality backlinks: Build high-quality backlinks from reputable websites that are relevant to your niche.\n\n8. Use social media: Use social media to promote your content and build your online presence, which can help improve your search engine rankings.\n\n9. Make sure your website is mobile-friendly: Ensure that your website is optimized for mobile devices, as mobile-friendly websites tend to rank higher in search engine results.\n\nBy following these tips, you can optimize your website for search engines and improve your visibility in search engine results pages (SERPs)."}, {"role": "user", "content": "How can I improve my website's accessibility for people with disabilities?"}, {"role": "assistant", "content": "Here are some ways to improve your website's accessibility for people with disabilities:\n\n1. Use descriptive and meaningful text for links: Use descriptive and meaningful text for links to help users understand where the link goes and what it does.\n\n2. Provide alternative text for images: Provide alternative text for images to help users with visual impairments understand the content of the image.\n\n3. Use headings and subheadings: Use headings and subheadings to structure your content and make it easier for screen readers to read.\n\n4. Use color with care: Use color with care and ensure that color is not the only way to convey information or distinguish between different elements on your website.\n\n5. Provide transcripts for audio and video content: Provide transcripts for audio and video content to help users with hearing impairments understand the content.\n\n6. Use clear and concise language: Use clear and concise language to make your content easy to understand for users with cognitive disabilities.\n\n7. Ensure the website is keyboard accessible: Ensure that your website is keyboard accessible to help users with motor disabilities navigate your website.\n\n8. Make sure your website is responsive: Ensure that your website is responsive and can be easily accessed on different devices and screen sizes.\n\n9. Use accessibility tools: Use accessibility tools such as screen readers, screen magnifiers, and other assistive technologies to test your website's accessibility.\n\nBy following these tips, you can make your website more accessible to people with disabilities and provide a better user experience for all users."}]} {"system": "You are a helpful assistant.", "conversations": [{"role": "user", "content": "What steps should I take to reduce taxes on my income?"}, {"role": "assistant", "content": "There are several steps you can take to reduce taxes on your income:\n\n1. Contribute to a retirement account: Contributions to a traditional IRA or a 401(k) can reduce your taxable income and save you money on taxes.\n\n2. Itemize your deductions: If you have significant expenses such as mortgage interest, medical expenses, or charitable contributions, itemizing your deductions may reduce your taxable income.\n\n3. Take advantage of tax credits: Tax credits, such as the Earned Income Tax Credit or the Child Tax Credit, can reduce your tax bill dollar for dollar.\n\n4. Invest in tax-free or tax-deferred accounts: Investments in tax-free or tax-deferred accounts such as municipal bonds, Roth IRAs, or 529 plans can reduce your taxable income.\n\n5. Consider income splitting: If you are married, consider splitting income with your spouse to reduce your overall tax bill.\n\nIt\u2019s important to consult with a tax professional or financial advisor to determine the best strategy for your specific situation."}, {"role": "user", "content": "Can you give me an example of a tax credit that can help reduce my taxes?"}, {"role": "assistant", "content": "Sure! One example of a tax credit that can help reduce your taxes is the Child Tax Credit. This is a credit of up to $2,000 per child under the age of 17 that can be claimed on your tax return. To qualify for the full credit, your income must be below a certain threshold (currently $400,000 for married filing jointly, or $200,000 for all other filers).\n\nIf your income is above the threshold, you may still be eligible for a partial credit. The credit is partially refundable, meaning that if the credit exceeds your tax liability, you may receive a refund for the difference up to $1,400 per child.\n\nTo claim the Child Tax Credit, you must have a qualifying child who meets certain criteria, such as being a U.S. citizen, living with you for more than half the year, and being related to you.\n\nIt\u2019s important to note that tax laws can change, so it\u2019s always a good idea to check with a tax professional or use tax preparation software to ensure you are claiming all the credits and deductions you are eligible for."}, {"role": "user", "content": "Can you explain the difference between a traditional IRA and a Roth IRA?"}, {"role": "assistant", "content": "Yes, I can explain the difference between a traditional IRA and a Roth IRA.\n\nA traditional IRA is a retirement account in which you can make pre-tax contributions, meaning you can deduct your contribution from your taxable income in the year you make it. The money in your traditional IRA grows tax-deferred, which means you won\u2019t pay taxes on the earnings until you withdraw the money in retirement. When you withdraw money from a traditional IRA, it is taxed as ordinary income.\n\nA Roth IRA, on the other hand, is a retirement account in which you make after-tax contributions, meaning you don\u2019t get a tax deduction for your contribution in the year you make it. However, the money in your Roth IRA grows tax-free, which means you won\u2019t pay taxes on the earnings when you withdraw the money in retirement. Additionally, qualified withdrawals from a Roth IRA are tax-free.\n\nThe difference between the two types of accounts is primarily when you pay taxes: with a traditional IRA, you pay taxes on the money when you withdraw it in retirement, while with a Roth IRA, you pay taxes on the money when you contribute to the account.\n\nIt\u2019s important to note that both types of accounts have contribution limits and other rules, so it\u2019s a good idea to consult with a financial advisor or tax professional to determine which type of account is best for your individual situation."}, {"role": "user", "content": "What is the contribution limit for a Roth IRA in 2021?"}, {"role": "assistant", "content": "The contribution limit for a Roth IRA in 2021 is $6,000 for individuals under age 50, and $7,000 for those age 50 and above. This means that you can contribute up to $6,000 or $7,000 (depending on your age) per year to your Roth IRA, as long as your income falls below certain limits.\n\nIt\u2019s important to note that contribution limits may change from year to year, so it\u2019s always a good idea to check the current year\u2019s limits before making any contributions. Additionally, there are income limits that determine whether you are eligible to contribute to a Roth IRA, and those limits can also change from year to year.\n\nIf you have any questions about Roth IRA contributions or eligibility, it\u2019s a good idea to consult with a financial advisor or tax professional."}, {"role": "user", "content": "Can you explain the difference between a 401(k) and an IRA?"}, {"role": "assistant", "content": "Yes, I can explain the difference between a 401(k) and an IRA.\n\nA 401(k) is an employer-sponsored retirement plan that allows employees to contribute a portion of their pre-tax salary to the plan. Employers may also offer matching contributions up to a certain amount. The money in a 401(k) grows tax-deferred, meaning you won\u2019t pay taxes on the earnings until you withdraw the money in retirement. Contributions to a 401(k) are subject to annual limits, which are set by the IRS.\n\nAn IRA, or individual retirement account, is a retirement account that you can set up on your own, outside of an employer-sponsored plan. IRAs come in two main types: traditional and Roth. A traditional IRA allows you to make pre-tax contributions, meaning you can deduct your contribution from your taxable income in the year you make it. The money in a traditional IRA grows tax-deferred, and you\u2019ll pay taxes on the money when you withdraw it in retirement. With a Roth IRA, you make after-tax contributions, meaning you don\u2019t get a tax deduction for your contribution in the year you make it. The money in a Roth IRA grows tax-free, and qualified withdrawals from a Roth IRA are tax-free.\n\nThe main difference between a 401(k) and an IRA is that a 401(k) is an employer-sponsored plan, while an IRA is an individual retirement account that you set up on your own. Additionally, 401(k) plans may have higher contribution limits than IRAs, and employers may offer matching contributions or other incentives to encourage employees to participate in the plan.\n\nIt\u2019s a good idea to consult with a financial advisor or tax professional to determine which type of account is best for your individual situation."}]} -{"system": "You are a helpful assistant.", "conversations": [{"role": "user", "content": "What strategies should I use to focus better while studying?"}, {"role": "assistant", "content": "Here are some strategies that can help you focus better while studying:\n\n1. Create a conducive environment: Find a quiet, distraction-free location to study. Make sure the lighting is appropriate, and the temperature is comfortable.\n\n2. Set a study schedule: Create a study schedule that works for you and stick to it. This helps you stay focused and avoids procrastination.\n\n3. Take breaks: Taking short breaks between study sessions can help you stay focused and retain information better.\n\n4. Eliminate distractions: Turn off your phone, log out from social media, and avoid any other distractions that can prevent you from concentrating.\n\n5. Use active learning techniques: Use active learning techniques such as note-taking, summarizing, and teaching others to help you stay engaged.\n\n6. Stay motivated: Set goals for yourself and reward yourself when you achieve them. This can help you stay motivated and focused.\n\n7. Get enough sleep and exercise: A healthy lifestyle can help you stay alert and focused while studying. Make sure you get enough sleep and exercise regularly.\n\nRemember, every individual is different, and what works for one person may not work for another. Find what works best for you and stick to it."}, {"role": "user", "content": "Can you give me some examples of active learning techniques that I can use while studying?"}, {"role": "assistant", "content": "Sure! Here are some examples of active learning techniques that you can use while studying:\n\n1. Summarizing: Summarize the main ideas in your own words. This helps you retain the information better.\n\n2. Note-taking: Take notes while you\u2019re reading or listening to lectures. This helps you focus your attention and remember the key points.\n\n3. Teaching: Explain the concepts you\u2019ve learned to someone else. This helps you understand the material better and identify areas where you need further clarification.\n\n4. Practice questions: Complete practice questions or quizzes to test your understanding of the material.\n\n5. Mnemonics: Use mnemonics (memory aids) to help you remember important information. For example, using acronyms or visual associations.\n\n6. Group study: Study with a group of peers and share your knowledge with each other. This helps you learn from different perspectives and keeps you motivated.\n\n7. Mind mapping: Create visual diagrams or mind maps to organize information and understand the relationships between different concepts.\n\nRemember, using a combination of different active learning techniques can help you to better understand and retain information."}, {"role": "user", "content": "How can I incorporate active learning techniques while studying for math?"}, {"role": "assistant", "content": "Math is a subject that requires a lot of practice, so incorporating active learning techniques can be very helpful in mastering the concepts. Here are some ways you can use active learning techniques while studying for math:\n\n1. Practice problems: Math requires a lot of practice to master the concepts. Solve practice problems and work through examples to reinforce your understanding of the material.\n\n2. Explain concepts: Explain the math concepts you\u2019ve learned to someone else. This helps you solidify your understanding and identify areas where you need further clarification.\n\n3. Create flashcards: Create flashcards for formulas, equations, and definitions. Use them to quiz yourself and help you memorize the material.\n\n4. Write out explanations: Write out explanations of the steps you took to solve a problem. This helps you identify areas where you may have made mistakes and reinforce your understanding of the material.\n\n5. Work in groups: Study with a group of peers and work through problems together. This helps you learn from different perspectives and keeps you motivated.\n\n6. Use online resources: There are many online resources available for math, including interactive tutorials and videos. Use them to reinforce your understanding of the material.\n\nRemember, math can be challenging, but incorporating active learning techniques can help you to better understand and retain the material."}, {"role": "user", "content": "How can I manage my time effectively while studying for multiple exams?"}, {"role": "assistant", "content": "Managing your time effectively while studying for multiple exams can be challenging, but here are some tips that can help:\n\n1. Create a study schedule: Make a schedule that outlines your study sessions for each subject. Consider your other commitments and allocate sufficient time for each subject.\n\n2. Prioritize the most important exams: Determine which exams are the most important and allocate more study time to these subjects.\n\n3. Break down the material: Break down the material into smaller, manageable chunks. This makes it easier to study and makes the task less overwhelming.\n\n4. Use study aids: Use study aids such as flashcards or summary notes to help you study more efficiently.\n\n5. Avoid multitasking: Avoid multitasking and focus on one subject at a time. This helps you to concentrate better and retain information more effectively.\n\n6. Take breaks: Taking regular breaks between study sessions can help you to stay focused and avoid burnout.\n\n7. Review regularly: Regularly review your notes and study materials to reinforce your understanding of the material.\n\n8. Stay organized: Keep your study materials organized and easily accessible. This saves time and helps you to focus on studying rather than searching for materials.\n\nRemember, effective time management is crucial when studying for multiple exams. By creating a study schedule, breaking down the material, prioritizing tasks, and staying organized, you can manage your time effectively and achieve better results."}]} \ No newline at end of file +{"system": "You are a helpful assistant.", "conversations": [{"role": "user", "content": "What strategies should I use to focus better while studying?"}, {"role": "assistant", "content": "Here are some strategies that can help you focus better while studying:\n\n1. Create a conducive environment: Find a quiet, distraction-free location to study. Make sure the lighting is appropriate, and the temperature is comfortable.\n\n2. Set a study schedule: Create a study schedule that works for you and stick to it. This helps you stay focused and avoids procrastination.\n\n3. Take breaks: Taking short breaks between study sessions can help you stay focused and retain information better.\n\n4. Eliminate distractions: Turn off your phone, log out from social media, and avoid any other distractions that can prevent you from concentrating.\n\n5. Use active learning techniques: Use active learning techniques such as note-taking, summarizing, and teaching others to help you stay engaged.\n\n6. Stay motivated: Set goals for yourself and reward yourself when you achieve them. This can help you stay motivated and focused.\n\n7. Get enough sleep and exercise: A healthy lifestyle can help you stay alert and focused while studying. Make sure you get enough sleep and exercise regularly.\n\nRemember, every individual is different, and what works for one person may not work for another. Find what works best for you and stick to it."}, {"role": "user", "content": "Can you give me some examples of active learning techniques that I can use while studying?"}, {"role": "assistant", "content": "Sure! Here are some examples of active learning techniques that you can use while studying:\n\n1. Summarizing: Summarize the main ideas in your own words. This helps you retain the information better.\n\n2. Note-taking: Take notes while you\u2019re reading or listening to lectures. This helps you focus your attention and remember the key points.\n\n3. Teaching: Explain the concepts you\u2019ve learned to someone else. This helps you understand the material better and identify areas where you need further clarification.\n\n4. Practice questions: Complete practice questions or quizzes to test your understanding of the material.\n\n5. Mnemonics: Use mnemonics (memory aids) to help you remember important information. For example, using acronyms or visual associations.\n\n6. Group study: Study with a group of peers and share your knowledge with each other. This helps you learn from different perspectives and keeps you motivated.\n\n7. Mind mapping: Create visual diagrams or mind maps to organize information and understand the relationships between different concepts.\n\nRemember, using a combination of different active learning techniques can help you to better understand and retain information."}, {"role": "user", "content": "How can I incorporate active learning techniques while studying for math?"}, {"role": "assistant", "content": "Math is a subject that requires a lot of practice, so incorporating active learning techniques can be very helpful in mastering the concepts. Here are some ways you can use active learning techniques while studying for math:\n\n1. Practice problems: Math requires a lot of practice to master the concepts. Solve practice problems and work through examples to reinforce your understanding of the material.\n\n2. Explain concepts: Explain the math concepts you\u2019ve learned to someone else. This helps you solidify your understanding and identify areas where you need further clarification.\n\n3. Create flashcards: Create flashcards for formulas, equations, and definitions. Use them to quiz yourself and help you memorize the material.\n\n4. Write out explanations: Write out explanations of the steps you took to solve a problem. This helps you identify areas where you may have made mistakes and reinforce your understanding of the material.\n\n5. Work in groups: Study with a group of peers and work through problems together. This helps you learn from different perspectives and keeps you motivated.\n\n6. Use online resources: There are many online resources available for math, including interactive tutorials and videos. Use them to reinforce your understanding of the material.\n\nRemember, math can be challenging, but incorporating active learning techniques can help you to better understand and retain the material."}, {"role": "user", "content": "How can I manage my time effectively while studying for multiple exams?"}, {"role": "assistant", "content": "Managing your time effectively while studying for multiple exams can be challenging, but here are some tips that can help:\n\n1. Create a study schedule: Make a schedule that outlines your study sessions for each subject. Consider your other commitments and allocate sufficient time for each subject.\n\n2. Prioritize the most important exams: Determine which exams are the most important and allocate more study time to these subjects.\n\n3. Break down the material: Break down the material into smaller, manageable chunks. This makes it easier to study and makes the task less overwhelming.\n\n4. Use study aids: Use study aids such as flashcards or summary notes to help you study more efficiently.\n\n5. Avoid multitasking: Avoid multitasking and focus on one subject at a time. This helps you to concentrate better and retain information more effectively.\n\n6. Take breaks: Taking regular breaks between study sessions can help you to stay focused and avoid burnout.\n\n7. Review regularly: Regularly review your notes and study materials to reinforce your understanding of the material.\n\n8. Stay organized: Keep your study materials organized and easily accessible. This saves time and helps you to focus on studying rather than searching for materials.\n\nRemember, effective time management is crucial when studying for multiple exams. By creating a study schedule, breaking down the material, prioritizing tasks, and staying organized, you can manage your time effectively and achieve better results."}]} diff --git a/full_automation.py b/full_automation.py index 5044661..361a4b4 100644 --- a/full_automation.py +++ b/full_automation.py @@ -64,14 +64,18 @@ secret_key=upload_data["data"]["secret_key"], endpoint_url=upload_data["data"]["endpoint_url"], session_token=upload_data["data"]["session_token"], - bucket=upload_data["data"]["bucket"] + bucket=upload_data["data"]["bucket"], ) cf_storage.initialize() - cf_storage.upload_folder(local_folder="outputs", - cloud_path=upload_data["data"]["folder_name"]) + cf_storage.upload_folder( + local_folder="outputs", cloud_path=upload_data["data"]["folder_name"] + ) submit_task( - task_id, model2base_model[model_id], gpu_type, - upload_data["data"]["bucket"], upload_data["data"]["folder_name"] + task_id, + model2base_model[model_id], + gpu_type, + upload_data["data"]["bucket"], + upload_data["data"]["folder_name"], ) logger.info("Task submitted successfully") except Exception as e: diff --git a/utils/cloudflare_utils.py b/utils/cloudflare_utils.py index 1721e2c..981e5cb 100644 --- a/utils/cloudflare_utils.py +++ b/utils/cloudflare_utils.py @@ -17,13 +17,22 @@ def __call__(self, bytes_amount): with self._lock: self._seen_so_far += bytes_amount percentage = (self._seen_so_far / self.size) * 100 - logger.info("\r%s %s / %s (%.2f%%)" % ( - self.filename, self._seen_so_far, self.size, - percentage), end="") + logger.info( + "\r%s %s / %s (%.2f%%)" + % (self.filename, self._seen_so_far, self.size, percentage), + end="", + ) class CloudStorage: - def __init__(self, access_key=None, secret_key=None, endpoint_url=None, bucket=None, session_token=None): + def __init__( + self, + access_key=None, + secret_key=None, + endpoint_url=None, + bucket=None, + session_token=None, + ): self.access_key = access_key self.secret_key = secret_key self.endpoint_url = endpoint_url @@ -32,15 +41,22 @@ def __init__(self, access_key=None, secret_key=None, endpoint_url=None, bucket=N self.session_token = session_token def initialize(self): - if self.access_key is None or self.secret_key is None or self.endpoint_url is None: - logger.error("Please provide access_key, secret_key, session_token and endpoint_url") + if ( + self.access_key is None + or self.secret_key is None + or self.endpoint_url is None + ): + logger.error( + "Please provide access_key, secret_key, session_token and endpoint_url" + ) raise - self.client = boto3.client('s3', - endpoint_url=self.endpoint_url, - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token - ) + self.client = boto3.client( + "s3", + endpoint_url=self.endpoint_url, + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + ) return self def upload_folder(self, local_folder, cloud_path, bucket=None): @@ -55,9 +71,13 @@ def upload_folder(self, local_folder, cloud_path, bucket=None): localFilePath = os.path.join(root, file) relativePath = os.path.relpath(localFilePath, local_folder) cloudPath = os.path.join(cloud_path, relativePath) - cloudPath = cloudPath.replace('\\', '/') + cloudPath = cloudPath.replace("\\", "/") try: - self.client.upload_file(localFilePath, bucket, cloudPath, - Callback=ProgressPercentage(localFilePath)) + self.client.upload_file( + localFilePath, + bucket, + cloudPath, + Callback=ProgressPercentage(localFilePath), + ) except botocore.exceptions.ClientError as e: logger.error(e) diff --git a/utils/flock_api.py b/utils/flock_api.py index e2fdfbf..8bc2bed 100644 --- a/utils/flock_api.py +++ b/utils/flock_api.py @@ -15,13 +15,12 @@ def get_task(task_id: int): def submit_task( - task_id: int, base_model: str, gpu_type: str, bucket: str, folder_name: str + task_id: int, base_model: str, gpu_type: str, bucket: str, folder_name: str ): payload = json.dumps( { "task_id": task_id, "data": { - "base_model": base_model, "gpu_type": gpu_type, "bucket": bucket, @@ -44,9 +43,7 @@ def submit_task( return response.json() -def get_address( - task_id: int -): +def get_address(task_id: int): payload = json.dumps( { "task_id": task_id, From 16f71d96cf595357acf4da292d3c52fe8cfdcc63 Mon Sep 17 00:00:00 2001 From: Nick W <52875892+nickcom007@users.noreply.github.com> Date: Tue, 31 Dec 2024 05:37:36 -0500 Subject: [PATCH 6/6] Update .pre-commit-config.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc79e09..57d62e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,13 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.6.4 - hooks: - # Run the linter. - - id: ruff - # Run the formatter. - - id: ruff-format + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.4 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format