From e6c27b50183bbe2d3b5d58236cd736c97bf33dae Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Thu, 30 Jan 2025 14:45:26 -0500 Subject: [PATCH 01/21] add print statements to test --- .github/workflows/readme.yml | 2 +- README.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index d3c1b08..8c3506d 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -5,7 +5,7 @@ name: README on: push: - branches: [ master ] + branches: [ '*' ] jobs: readme: diff --git a/README.md b/README.md index 3fd4247..f209c9a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,10 @@ In addtion, the `SigLLM` object takes in a `decimal` argument to determine how m ```python3 from sigllm import SigLLM +from mlblocks.discovery import get_primitives_paths, get_pipelines_paths + +print(get_primitives_paths()) +print(get_pipelines_paths()) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { From e47e92d463706a74d20078e5e6300c2ff7ffe568 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Thu, 30 Jan 2025 15:19:15 -0500 Subject: [PATCH 02/21] change entrypoint --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ef2c13a..3a04157 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,9 @@ dev = [ [project.urls] homepage = 'https://github.com/sintel-dev/sigllm/' -[project.entry-points] -mlblocks = { primitives = 'sigllm:MLBLOCKS_PRIMITIVES', pipelines = 'sigllm:MLBLOCKS_PIPELINES' } +[project.entry-points.mlblocks] +primitives = 'sigllm:MLBLOCKS_PRIMITIVES' +pipelines = 'sigllm:MLBLOCKS_PIPELINES' [build-system] requires = ['setuptools', 'wheel'] From 935efdbf08685d3fdca7f340085a5848d24c3ddb Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Sun, 2 Feb 2025 18:11:30 -0500 Subject: [PATCH 03/21] revert readme --- .github/workflows/readme.yml | 2 +- README.md | 4 ---- pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index 8c3506d..0b82c5b 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: python-version: ['3.10'] - os: [ubuntu-latest] + os: [macos-latest] steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index f209c9a..3fd4247 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,6 @@ In addtion, the `SigLLM` object takes in a `decimal` argument to determine how m ```python3 from sigllm import SigLLM -from mlblocks.discovery import get_primitives_paths, get_pipelines_paths - -print(get_primitives_paths()) -print(get_pipelines_paths()) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { diff --git a/pyproject.toml b/pyproject.toml index 3a04157..7d9e6d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,8 @@ dev = [ homepage = 'https://github.com/sintel-dev/sigllm/' [project.entry-points.mlblocks] -primitives = 'sigllm:MLBLOCKS_PRIMITIVES' pipelines = 'sigllm:MLBLOCKS_PIPELINES' +primitives = 'sigllm:MLBLOCKS_PRIMITIVES' [build-system] requires = ['setuptools', 'wheel'] From 53c9071f7098ec0a50c42b01f5cf58881996ddb1 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Sun, 2 Feb 2025 18:15:14 -0500 Subject: [PATCH 04/21] change python spec --- .github/workflows/readme.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index 0b82c5b..c59fe3e 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -12,12 +12,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.10'] - os: [macos-latest] + python-version: ['3.11'] + os: [ubuntu-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package and dependencies From d71a8f04154eacaf09f81dc98901fb7181f74628 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 12:59:48 -0500 Subject: [PATCH 05/21] manually add pipeline path --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 3fd4247..826b69b 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,14 @@ In this example we use `gpt_detector` pipeline and set some hyperparameters. In In addtion, the `SigLLM` object takes in a `decimal` argument to determine how many digits from the float value include. Here, we don't want to keep any decimal values, so we set it to zero. ```python3 +import os from sigllm import SigLLM +from mlblocks.discovery import get_pipelines_paths, add_pipelines_path + +print(get_pipelines_paths()) +path = os.path.join(os.getcwd(), '..', '..', 'sigllm', 'pipelines', 'detector') +add_pipelines_path(path) +print(get_pipelines_paths()) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { From 75d5bdd5b0f9966c33f652fec6d71d2ab60a0e70 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 13:04:51 -0500 Subject: [PATCH 06/21] add primitives manually too --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 826b69b..b3071e3 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,17 @@ In addtion, the `SigLLM` object takes in a `decimal` argument to determine how m import os from sigllm import SigLLM from mlblocks.discovery import get_pipelines_paths, add_pipelines_path +from mlblocks.discovery import get_primitives_paths, add_primitives_path print(get_pipelines_paths()) path = os.path.join(os.getcwd(), '..', '..', 'sigllm', 'pipelines', 'detector') add_pipelines_path(path) print(get_pipelines_paths()) +path = os.path.join(os.getcwd(), '..', '..', 'sigllm', 'primitives', 'jsons') +add_primitives_path(path) + + hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { "fixed_threshold": False From 97a6121dcc063ef47cf1d739a65cb659e0ff4519 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 13:26:16 -0500 Subject: [PATCH 07/21] add paths in tasks.py --- README.md | 12 ------------ tasks.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b3071e3..3fd4247 100644 --- a/README.md +++ b/README.md @@ -67,19 +67,7 @@ In this example we use `gpt_detector` pipeline and set some hyperparameters. In In addtion, the `SigLLM` object takes in a `decimal` argument to determine how many digits from the float value include. Here, we don't want to keep any decimal values, so we set it to zero. ```python3 -import os from sigllm import SigLLM -from mlblocks.discovery import get_pipelines_paths, add_pipelines_path -from mlblocks.discovery import get_primitives_paths, add_primitives_path - -print(get_pipelines_paths()) -path = os.path.join(os.getcwd(), '..', '..', 'sigllm', 'pipelines', 'detector') -add_pipelines_path(path) -print(get_pipelines_paths()) - -path = os.path.join(os.getcwd(), '..', '..', 'sigllm', 'primitives', 'jsons') -add_primitives_path(path) - hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { diff --git a/tasks.py b/tasks.py index 9eb8525..9c85992 100644 --- a/tasks.py +++ b/tasks.py @@ -10,6 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version +from mlblocks.discovery import add_primitives_path, add_pipelines_path def _get_minimum_versions(dependencies, python_version): min_versions = {} @@ -69,8 +70,21 @@ def minimum(c): unit(c) +def add_paths(): + base_path = os.path.join(os.getcwd(), 'sigllm') + primitives_path = os.path.join(base_path, 'primitives', 'jsons') + pipeline_paths = [ + os.path.join(base_path, 'pipelines', 'detector'), + os.path.join(base_path, 'pipelines', 'prompter') + ] + add_primitives_path(primitives_path) + for pipeline_path in pipeline_paths: + add_pipelines_path(pipeline_path) + + @task def readme(c): + add_paths() test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) From d42b4fcfb841c6f38c37be6fb488e79ce3bcd8c8 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 13:32:00 -0500 Subject: [PATCH 08/21] add Path function --- tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 9c85992..b6975f8 100644 --- a/tasks.py +++ b/tasks.py @@ -10,7 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from mlblocks.discovery import add_primitives_path, add_pipelines_path +from mlblocks.discovery import add_primitives_path, add_pipelines_path, get_primitives_paths def _get_minimum_versions(dependencies, python_version): min_versions = {} @@ -71,7 +71,7 @@ def minimum(c): def add_paths(): - base_path = os.path.join(os.getcwd(), 'sigllm') + base_path = Path(os.path.join(os.getcwd(), 'sigllm')) primitives_path = os.path.join(base_path, 'primitives', 'jsons') pipeline_paths = [ os.path.join(base_path, 'pipelines', 'detector'), @@ -85,6 +85,7 @@ def add_paths(): @task def readme(c): add_paths() + print(get_primitives_paths()) test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) From d2586da636b649c0f1fdf12741ab0ed79a308583 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 14:09:40 -0500 Subject: [PATCH 09/21] print pipeline paths --- tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index b6975f8..3ac8787 100644 --- a/tasks.py +++ b/tasks.py @@ -10,7 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from mlblocks.discovery import add_primitives_path, add_pipelines_path, get_primitives_paths +from mlblocks.discovery import add_primitives_path, add_pipelines_path, get_primitives_paths, get_pipelines_paths def _get_minimum_versions(dependencies, python_version): min_versions = {} @@ -86,6 +86,7 @@ def add_paths(): def readme(c): add_paths() print(get_primitives_paths()) + print(get_pipelines_paths()) test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) From d60af81a30c65f05db0870f5c6abbb2ca579b292 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 14:38:58 -0500 Subject: [PATCH 10/21] adding import --- tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks.py b/tasks.py index 3ac8787..aa4e382 100644 --- a/tasks.py +++ b/tasks.py @@ -10,6 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version +from sigllm import SigLLM from mlblocks.discovery import add_primitives_path, add_pipelines_path, get_primitives_paths, get_pipelines_paths def _get_minimum_versions(dependencies, python_version): From 1036e13d04497e40680c7fd9eb9b6504df97a966 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 14:50:50 -0500 Subject: [PATCH 11/21] test --- README.md | 3 +++ tasks.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3fd4247..2cd7915 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,9 @@ In addtion, the `SigLLM` object takes in a `decimal` argument to determine how m ```python3 from sigllm import SigLLM +from mlblocks.discovery import get_pipelines_paths + +print(get_pipelines_paths()) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { diff --git a/tasks.py b/tasks.py index aa4e382..41802b5 100644 --- a/tasks.py +++ b/tasks.py @@ -10,8 +10,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from sigllm import SigLLM -from mlblocks.discovery import add_primitives_path, add_pipelines_path, get_primitives_paths, get_pipelines_paths +from mlblocks.discovery import add_primitives_path, add_pipelines_path def _get_minimum_versions(dependencies, python_version): min_versions = {} @@ -85,6 +84,7 @@ def add_paths(): @task def readme(c): + print(os.walk(os.getcwd())) add_paths() print(get_primitives_paths()) print(get_pipelines_paths()) From ec235c65a4851d7fb24a7a62a0b175604ef63c48 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 14:54:40 -0500 Subject: [PATCH 12/21] remove print --- tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tasks.py b/tasks.py index 41802b5..0ff5053 100644 --- a/tasks.py +++ b/tasks.py @@ -86,8 +86,6 @@ def add_paths(): def readme(c): print(os.walk(os.getcwd())) add_paths() - print(get_primitives_paths()) - print(get_pipelines_paths()) test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) From e50b51fc8a380fad886b09330bfd998b05e06236 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 15:06:08 -0500 Subject: [PATCH 13/21] copy json path --- tasks.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/tasks.py b/tasks.py index 0ff5053..e874812 100644 --- a/tasks.py +++ b/tasks.py @@ -70,30 +70,19 @@ def minimum(c): unit(c) -def add_paths(): - base_path = Path(os.path.join(os.getcwd(), 'sigllm')) - primitives_path = os.path.join(base_path, 'primitives', 'jsons') - pipeline_paths = [ - os.path.join(base_path, 'pipelines', 'detector'), - os.path.join(base_path, 'pipelines', 'prompter') - ] - add_primitives_path(primitives_path) - for pipeline_path in pipeline_paths: - add_pipelines_path(pipeline_path) - - @task def readme(c): - print(os.walk(os.getcwd())) - add_paths() + pipeline_path = 'sigllm/pipelines/detector/gpt_detector.json' test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) cwd = os.getcwd() os.makedirs(test_path, exist_ok=True) + os.makedirs(test_path / 'mlpipelines', exist_ok=True) shutil.copy('README.md', test_path / 'README.md') shutil.copy('tutorials/data.csv', test_path / 'data.csv') + shutil.copy(pipeline_path, test_path / 'mlpipelines' / 'gpt_detector.json') os.chdir(test_path) c.run('rundoc run --single-session python3 -t python3 README.md') os.chdir(cwd) From f0d1233837cfa86d49b1b84f5ef54e46d19e3a0e Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:05:43 -0500 Subject: [PATCH 14/21] print contents --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cd7915..182c7cf 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,9 @@ In addtion, the `SigLLM` object takes in a `decimal` argument to determine how m from sigllm import SigLLM from mlblocks.discovery import get_pipelines_paths -print(get_pipelines_paths()) +for path in get_pipelines_paths(): + print(path) + print(os.listdir(path)) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { From 09064f9e8c82bdb5eccc0e6975c8bad0bd3d0ccd Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:23:30 -0500 Subject: [PATCH 15/21] print contents --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 182c7cf..9985330 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ In this example we use `gpt_detector` pipeline and set some hyperparameters. In In addtion, the `SigLLM` object takes in a `decimal` argument to determine how many digits from the float value include. Here, we don't want to keep any decimal values, so we set it to zero. ```python3 +import os from sigllm import SigLLM from mlblocks.discovery import get_pipelines_paths From 1e1e7e30d4c159da137c36b5996f1b4252cfd09e Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:29:12 -0500 Subject: [PATCH 16/21] print contents --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9985330..b0ae781 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,8 @@ from mlblocks.discovery import get_pipelines_paths for path in get_pipelines_paths(): print(path) - print(os.listdir(path)) + if os.path.isdir(path): + print(os.listdir(path)) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { From ced9a2ae325b6312ec56a2d0417d43962f57e936 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:43:08 -0500 Subject: [PATCH 17/21] keep manifest --- MANIFEST.in | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..cdebd58 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.md +include LICENSE +include README.md + +recursive-include sigllm *.json + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.md *.rst conf.py Makefile make.bat *.jpg *.png *.gif From 0bfbd444702bb1109e5507e2ccf3849be91212d8 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:51:42 -0500 Subject: [PATCH 18/21] restore original tests --- MANIFEST.in | 13 ------ README.md | 7 ---- pyproject.toml | 16 ++++++- tasks.py | 111 +++++++++++++++++++++++++------------------------ 4 files changed, 71 insertions(+), 76 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cdebd58..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,13 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.md -include LICENSE -include README.md - -recursive-include sigllm *.json - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.md *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/README.md b/README.md index b0ae781..3fd4247 100644 --- a/README.md +++ b/README.md @@ -67,14 +67,7 @@ In this example we use `gpt_detector` pipeline and set some hyperparameters. In In addtion, the `SigLLM` object takes in a `decimal` argument to determine how many digits from the float value include. Here, we don't want to keep any decimal values, so we set it to zero. ```python3 -import os from sigllm import SigLLM -from mlblocks.discovery import get_pipelines_paths - -for path in get_pipelines_paths(): - print(path) - if os.path.isdir(path): - print(os.listdir(path)) hyperparameters = { "orion.primitives.timeseries_anomalies.find_anomalies#1": { diff --git a/pyproject.toml b/pyproject.toml index 7d9e6d5..6686cbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,21 @@ include = ['sigllm', 'sigllm.*'] namespaces = false [tool.setuptools.package-data] -"*" = ["*.*"] +'*' = [ + 'CONTRIBUTING.rst', + 'HISTORY.md', + 'README.md', + '*.md', + '*.rst', + '*.json', + 'conf.py', + 'Makefile', + 'make.bat', + '*.jpg', + '*.png', + '*.gif', +] + [tool.setuptools.exclude-package-data] '*' = [ diff --git a/tasks.py b/tasks.py index e874812..bede560 100644 --- a/tasks.py +++ b/tasks.py @@ -1,66 +1,77 @@ import glob +import operator import os +import pkg_resources +import platform +import re import shutil import stat -import sys from pathlib import Path -import tomli from invoke import task -from packaging.requirements import Requirement -from packaging.version import Version -from mlblocks.discovery import add_primitives_path, add_pipelines_path -def _get_minimum_versions(dependencies, python_version): - min_versions = {} - for dependency in dependencies: - if '@' in dependency: - name, url = dependency.split(' @ ') - min_versions[name] = f'{url}#egg={name}' - continue +COMPARISONS = { + '>=': operator.ge, + '>': operator.gt, + '<': operator.lt, + '<=': operator.le +} - req = Requirement(dependency) - if ';' in dependency: - marker = req.marker - if marker and not marker.evaluate({'python_version': python_version}): - continue # python version does not match - if req.name not in min_versions: - min_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), None) - if min_version: - min_versions[req.name] = f'{req.name}=={min_version}' - - elif '@' not in min_versions[req.name]: - existing_version = Version(min_versions[req.name].split('==')[1]) - new_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), existing_version) - if new_version > existing_version: - min_versions[req.name] = f'{req.name}=={new_version}' - - return list(min_versions.values()) +@task +def check_dependencies(c): + c.run('python -m pip check') @task -def install_minimum(c): - with open('pyproject.toml', 'rb') as pyproject_file: - pyproject_data = tomli.load(pyproject_file) +def unit(c): + c.run('python -m pytest --cov=sigllm --cov-report=xml') - dependencies = pyproject_data.get('project', {}).get('dependencies', []) - python_version = '.'.join(map(str, sys.version_info[:2])) - minimum_versions = _get_minimum_versions(dependencies, python_version) - if minimum_versions: - c.run(f'python -m pip install {" ".join(minimum_versions)}') +def _validate_python_version(line): + is_valid = True + for python_version_match in re.finditer(r"python_version(<=?|>=?|==)\'(\d\.?)+\'", line): + python_version = python_version_match.group(0) + comparison = re.search(r'(>=?|<=?|==)', python_version).group(0) + version_number = python_version.split(comparison)[-1].replace("'", "") + comparison_function = COMPARISONS[comparison] + is_valid = is_valid and comparison_function( + pkg_resources.parse_version(platform.python_version()), + pkg_resources.parse_version(version_number), + ) - -@task -def check_dependencies(c): - c.run('python -m pip check') + return is_valid @task -def unit(c): - c.run('python -m pytest --cov=sigllm --cov-report=xml') +def install_minimum(c): + with open('setup.py', 'r') as setup_py: + lines = setup_py.read().splitlines() + + versions = [] + started = False + for line in lines: + if started: + if line == ']': + started = False + continue + + line = line.strip() + if _validate_python_version(line): + requirement = re.match(r'[^>]*', line).group(0) + requirement = re.sub(r"""['",]""", '', requirement) + version = re.search(r'>=?(\d\.?)+\w*', line).group(0) + if version: + version = re.sub(r'>=?', '==', version) + version = re.sub(r"""['",]""", '', version) + requirement += version + versions.append(requirement) + + elif (line.startswith('install_requires = [')): + started = True + + c.run(f'python -m pip install {" ".join(versions)}') @task @@ -72,17 +83,14 @@ def minimum(c): @task def readme(c): - pipeline_path = 'sigllm/pipelines/detector/gpt_detector.json' test_path = Path('tests/readme_test') if test_path.exists() and test_path.is_dir(): shutil.rmtree(test_path) cwd = os.getcwd() os.makedirs(test_path, exist_ok=True) - os.makedirs(test_path / 'mlpipelines', exist_ok=True) shutil.copy('README.md', test_path / 'README.md') shutil.copy('tutorials/data.csv', test_path / 'data.csv') - shutil.copy(pipeline_path, test_path / 'mlpipelines' / 'gpt_detector.json') os.chdir(test_path) c.run('rundoc run --single-session python3 -t python3 README.md') os.chdir(cwd) @@ -102,15 +110,8 @@ def tutorials(c): @task def lint(c): check_dependencies(c) - c.run('ruff check .') - c.run('ruff format --check --diff .') - - -@task -def fix_lint(c): - check_dependencies(c) - c.run('ruff check --fix .') - c.run('ruff format .') + c.run('flake8 sigllm tests') + c.run('isort -c --recursive sigllm tests') def remove_readonly(func, path, _): From 24d1dc9cb882c426ce90d922ae750a50ec02a705 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 16:57:18 -0500 Subject: [PATCH 19/21] keep manifest and defaults for package-data --- MANIFEST.in | 13 +++++++++++++ pyproject.toml | 16 +--------------- 2 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..cdebd58 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.md +include LICENSE +include README.md + +recursive-include sigllm *.json + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.md *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/pyproject.toml b/pyproject.toml index 6686cbb..0a2cf5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,21 +102,7 @@ include = ['sigllm', 'sigllm.*'] namespaces = false [tool.setuptools.package-data] -'*' = [ - 'CONTRIBUTING.rst', - 'HISTORY.md', - 'README.md', - '*.md', - '*.rst', - '*.json', - 'conf.py', - 'Makefile', - 'make.bat', - '*.jpg', - '*.png', - '*.gif', -] - +'*' = ['*.*'] [tool.setuptools.exclude-package-data] '*' = [ From be5184a260e6fcd12fa53707784cccb28f3b0bb0 Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Mon, 3 Feb 2025 17:23:10 -0500 Subject: [PATCH 20/21] update tasks.py --- tasks.py | 107 ++++++++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/tasks.py b/tasks.py index bede560..9eb8525 100644 --- a/tasks.py +++ b/tasks.py @@ -1,77 +1,65 @@ import glob -import operator import os -import pkg_resources -import platform -import re import shutil import stat +import sys from pathlib import Path +import tomli from invoke import task +from packaging.requirements import Requirement +from packaging.version import Version -COMPARISONS = { - '>=': operator.ge, - '>': operator.gt, - '<': operator.lt, - '<=': operator.le -} +def _get_minimum_versions(dependencies, python_version): + min_versions = {} + for dependency in dependencies: + if '@' in dependency: + name, url = dependency.split(' @ ') + min_versions[name] = f'{url}#egg={name}' + continue + req = Requirement(dependency) + if ';' in dependency: + marker = req.marker + if marker and not marker.evaluate({'python_version': python_version}): + continue # python version does not match -@task -def check_dependencies(c): - c.run('python -m pip check') + if req.name not in min_versions: + min_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), None) + if min_version: + min_versions[req.name] = f'{req.name}=={min_version}' + + elif '@' not in min_versions[req.name]: + existing_version = Version(min_versions[req.name].split('==')[1]) + new_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), existing_version) + if new_version > existing_version: + min_versions[req.name] = f'{req.name}=={new_version}' + + return list(min_versions.values()) @task -def unit(c): - c.run('python -m pytest --cov=sigllm --cov-report=xml') +def install_minimum(c): + with open('pyproject.toml', 'rb') as pyproject_file: + pyproject_data = tomli.load(pyproject_file) + dependencies = pyproject_data.get('project', {}).get('dependencies', []) + python_version = '.'.join(map(str, sys.version_info[:2])) + minimum_versions = _get_minimum_versions(dependencies, python_version) -def _validate_python_version(line): - is_valid = True - for python_version_match in re.finditer(r"python_version(<=?|>=?|==)\'(\d\.?)+\'", line): - python_version = python_version_match.group(0) - comparison = re.search(r'(>=?|<=?|==)', python_version).group(0) - version_number = python_version.split(comparison)[-1].replace("'", "") - comparison_function = COMPARISONS[comparison] - is_valid = is_valid and comparison_function( - pkg_resources.parse_version(platform.python_version()), - pkg_resources.parse_version(version_number), - ) + if minimum_versions: + c.run(f'python -m pip install {" ".join(minimum_versions)}') - return is_valid + +@task +def check_dependencies(c): + c.run('python -m pip check') @task -def install_minimum(c): - with open('setup.py', 'r') as setup_py: - lines = setup_py.read().splitlines() - - versions = [] - started = False - for line in lines: - if started: - if line == ']': - started = False - continue - - line = line.strip() - if _validate_python_version(line): - requirement = re.match(r'[^>]*', line).group(0) - requirement = re.sub(r"""['",]""", '', requirement) - version = re.search(r'>=?(\d\.?)+\w*', line).group(0) - if version: - version = re.sub(r'>=?', '==', version) - version = re.sub(r"""['",]""", '', version) - requirement += version - versions.append(requirement) - - elif (line.startswith('install_requires = [')): - started = True - - c.run(f'python -m pip install {" ".join(versions)}') +def unit(c): + c.run('python -m pytest --cov=sigllm --cov-report=xml') @task @@ -110,8 +98,15 @@ def tutorials(c): @task def lint(c): check_dependencies(c) - c.run('flake8 sigllm tests') - c.run('isort -c --recursive sigllm tests') + c.run('ruff check .') + c.run('ruff format --check --diff .') + + +@task +def fix_lint(c): + check_dependencies(c) + c.run('ruff check --fix .') + c.run('ruff format .') def remove_readonly(func, path, _): From cbf86bbe1f4d15d4ebd19ace0750a00501a2c28c Mon Sep 17 00:00:00 2001 From: Sarah Alnegheimish Date: Tue, 4 Feb 2025 14:13:42 -0500 Subject: [PATCH 21/21] test on master only --- .github/workflows/readme.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index c59fe3e..8d43535 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -5,14 +5,14 @@ name: README on: push: - branches: [ '*' ] + branches: [ master ] jobs: readme: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.11'] + python-version: ['3.10'] os: [ubuntu-latest] steps: - uses: actions/checkout@v4