From 550bf05b266624a5b3a596bfcc6719877c3ea432 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 19 Dec 2025 00:03:21 +0000 Subject: [PATCH] chore: update all CI files --- .ci/release | 69 +++++++++++------------------ .ci/release-uv | 56 ----------------------- .github/workflows/main.yml | 8 ++-- conftest.py | 57 ------------------------ demo.py | 12 +---- pyproject.toml | 6 +-- pytest.ini | 5 ++- src/my/coding/commits.py | 2 +- src/my/core/_deprecated/dataset.py | 2 +- src/my/core/cachew.py | 4 +- src/my/core/cfg.py | 5 ++- src/my/core/common.py | 2 +- src/my/core/core_config.py | 2 +- src/my/core/influxdb.py | 1 - src/my/core/kompress.py | 4 +- src/my/core/orgmode.py | 2 +- src/my/core/pandas.py | 4 +- src/my/core/query.py | 2 +- src/my/core/stats.py | 2 +- src/my/core/tests/test_get_files.py | 4 +- src/my/core/utils/itertools.py | 15 ++++--- src/my/fbmessenger/__init__.py | 1 - src/my/fbmessenger/all.py | 3 ++ src/my/fbmessenger/common.py | 1 + src/my/foursquare.py | 4 +- src/my/github/all.py | 3 +- src/my/github/common.py | 5 ++- src/my/goodreads.py | 13 ++++-- src/my/hypothesis.py | 1 + src/my/instagram/all.py | 6 +++ src/my/ip/all.py | 4 +- src/my/jawbone/__init__.py | 25 ++++++----- tox.ini | 20 +++------ 33 files changed, 118 insertions(+), 232 deletions(-) delete mode 100755 .ci/release-uv delete mode 100644 conftest.py diff --git a/.ci/release b/.ci/release index 6cff663e..fbb74c00 100755 --- a/.ci/release +++ b/.ci/release @@ -1,65 +1,48 @@ #!/usr/bin/env python3 ''' -Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]]. +Deploys Python package onto [[https://pypi.org][PyPi]] or [[https://test.pypi.org][test PyPi]]. -The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!). +- running manually -The script can be run manually. -It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on: -- every master commit, onto test pypi -- every new tag, onto production pypi + You'll need =UV_PUBLISH_TOKEN= env variable -You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]] -for Github Actions deployment to work. +- running on Github Actions + + Instead of env variable, relies on configuring github as Trusted publisher (https://docs.pypi.org/trusted-publishers/) -- both for test and regular pypi + + It's running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. + Packages are deployed on: + - every master commit, onto test pypi + - every new tag, onto production pypi ''' +UV_PUBLISH_TOKEN = 'UV_PUBLISH_TOKEN' + +import argparse import os -import sys from pathlib import Path from subprocess import check_call -import shutil is_ci = os.environ.get('CI') is not None + def main() -> None: - import argparse p = argparse.ArgumentParser() - p.add_argument('--test', action='store_true', help='use test pypi') + p.add_argument('--use-test-pypi', action='store_true') args = p.parse_args() - extra = [] - if args.test: - extra.extend(['--repository', 'testpypi']) + publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else [] root = Path(__file__).absolute().parent.parent - os.chdir(root) # just in case - - if is_ci: - # see https://github.com/actions/checkout/issues/217 - check_call('git fetch --prune --unshallow'.split()) - - dist = root / 'dist' - if dist.exists(): - shutil.rmtree(dist) - - check_call(['python3', '-m', 'build']) - - TP = 'TWINE_PASSWORD' - password = os.environ.get(TP) - if password is None: - print(f"WARNING: no {TP} passed", file=sys.stderr) - import pip_secrets - password = pip_secrets.token_test if args.test else pip_secrets.token # meh - - check_call([ - 'python3', '-m', 'twine', - 'upload', *dist.iterdir(), - *extra, - ], env={ - 'TWINE_USERNAME': '__token__', - TP: password, - **os.environ, - }) + os.chdir(root) # just in case + + check_call(['uv', 'build', '--clear']) + + if not is_ci: + # CI relies on trusted publishers so doesn't need env variable + assert UV_PUBLISH_TOKEN in os.environ, f'no {UV_PUBLISH_TOKEN} passed' + + check_call(['uv', 'publish', *publish_url]) if __name__ == '__main__': diff --git a/.ci/release-uv b/.ci/release-uv deleted file mode 100755 index 4da39b7f..00000000 --- a/.ci/release-uv +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -''' -Deploys Python package onto [[https://pypi.org][PyPi]] or [[https://test.pypi.org][test PyPi]]. - -- running manually - - You'll need =UV_PUBLISH_TOKEN= env variable - -- running on Github Actions - - Instead of env variable, relies on configuring github as Trusted publisher (https://docs.pypi.org/trusted-publishers/) -- both for test and regular pypi - - It's running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. - Packages are deployed on: - - every master commit, onto test pypi - - every new tag, onto production pypi -''' - -UV_PUBLISH_TOKEN = 'UV_PUBLISH_TOKEN' - -import argparse -import os -import shutil -from pathlib import Path -from subprocess import check_call - -is_ci = os.environ.get('CI') is not None - - -def main() -> None: - p = argparse.ArgumentParser() - p.add_argument('--use-test-pypi', action='store_true') - args = p.parse_args() - - publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else [] - - root = Path(__file__).absolute().parent.parent - os.chdir(root) # just in case - - # TODO ok, for now uv won't remove dist dir if it already exists - # https://github.com/astral-sh/uv/issues/10293 - dist = root / 'dist' - if dist.exists(): - shutil.rmtree(dist) - - check_call(['uv', 'build']) - - if not is_ci: - # CI relies on trusted publishers so doesn't need env variable - assert UV_PUBLISH_TOKEN in os.environ, f'no {UV_PUBLISH_TOKEN} passed' - - check_call(['uv', 'publish', *publish_url]) - - -if __name__ == '__main__': - main() diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ee26e29..ce2ca3bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,7 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: recursive fetch-depth: 0 # nicer to have all git history when debugging/for tests @@ -98,7 +98,7 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: recursive fetch-depth: 0 # pull all commits to correctly infer vcs version @@ -114,9 +114,9 @@ jobs: - name: 'release to test pypi' # always deploy merged master to test pypi if: github.event.ref == format('refs/heads/{0}', github.event.repository.master_branch) - run: .ci/release-uv --use-test-pypi + run: .ci/release --use-test-pypi - name: 'release to prod pypi' # always deploy tags to release pypi if: startsWith(github.event.ref, 'refs/tags/') - run: .ci/release-uv + run: .ci/release diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 895c4df0..00000000 --- a/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly -# without it, pytest can't discover the package root for some reason -# also see https://github.com/karlicoss/pytest_namespace_pkgs for more - -import os -import pathlib - -import _pytest.main -import _pytest.pathlib - -# we consider all dirs in repo/ to be namespace packages -root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src' -assert root_dir.exists(), root_dir - -# TODO assert it contains package name?? maybe get it via setuptools.. - -namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()] - -# resolve_package_path is called from _pytest.pathlib.import_path -# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem -resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path - - -def resolve_package_path(path: pathlib.Path) -> pathlib.Path | None: - result = path # search from the test file upwards - for parent in result.parents: - if str(parent) in namespace_pkg_dirs: - return parent - if os.name == 'nt': - # ??? for some reason on windows it is trying to call this against conftest? but not on linux/osx - if path.name == 'conftest.py': - return resolve_pkg_path_orig(path) - raise RuntimeError("Couldn't determine path for ", path) - - -# NOTE: seems like it's not necessary anymore? -# keeping it for now just in case -# after https://github.com/pytest-dev/pytest/pull/13426 we should be able to remove the whole conftest -# _pytest.pathlib.resolve_package_path = resolve_package_path - - -# without patching, the orig function returns just a package name for some reason -# (I think it's used as a sort of fallback) -# so we need to point it at the absolute path properly -# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. -search_pypath_orig = _pytest.main.search_pypath - - -def search_pypath(module_name: str) -> str: - mpath = root_dir / module_name.replace('.', os.sep) - if not mpath.is_dir(): - mpath = mpath.with_suffix('.py') - assert mpath.exists(), mpath # just in case - return str(mpath) - - -_pytest.main.search_pypath = search_pypath # ty: ignore[invalid-assignment] diff --git a/demo.py b/demo.py index 009d592d..a03bfb97 100755 --- a/demo.py +++ b/demo.py @@ -23,16 +23,8 @@ def run() -> None: ignore=ignore_patterns('.tox*'), # tox dir might have broken symlinks while tests are running in parallel ) - # 2. prepare repositories you'd be using. For this demo we only set up Hypothesis - tox = 'TOX' in os.environ - if tox: # tox doesn't like --user flag - check_call(f'{python} -m pip install git+https://github.com/karlicoss/hypexport.git'.split()) - else: - try: - import hypexport # noqa: F401 - except ModuleNotFoundError: - check_call(f'{python} -m pip --user git+https://github.com/karlicoss/hypexport.git'.split()) - + # 2. setup modules you'd be using. For this demo we only set up Hypothesis + check_call(f'{python} -m my.core module install my.hypothesis'.split()) # 3. prepare some demo Hypothesis data hypothesis_backups = Path('backups/hypothesis').resolve() diff --git a/pyproject.toml b/pyproject.toml index 5c88192d..56123108 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ optional = [ # On the other hand, it's a bit annoying that it's always included by default? # To make sure it's not included, need to use `uv run --exact --no-default-groups ...` testing = [ - "pytest", + "pytest>=9", # need version 9 for proper namespace package support "ruff", # used in some tests.. although shouldn't rely on it @@ -64,8 +64,8 @@ testing = [ typecheck = [ { include-group = "testing" }, "mypy", - "lxml", # for mypy coverage - "ty>=0.0.1a22", + "lxml", # for mypy html coverage + "ty==0.0.1a35", "HPI[optional]", "orgparse", # for my.core.orgmode diff --git a/pytest.ini b/pytest.ini index 226488bd..49e407fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,9 +2,12 @@ # discover files that don't follow test_ naming. Useful to keep tests along with the source code python_files = *.py -# this setting only impacts package/module naming under pytest, not the discovery +# this is necessary for --pyargs to discover implicit namespace packages correctly consider_namespace_packages = true +# see https://docs.pytest.org/en/stable/reference/reference.html#confval-strict +strict = true + addopts = # prevent pytest cache from being created... it craps into project dir and I never use it anyway -p no:cacheprovider diff --git a/src/my/coding/commits.py b/src/my/coding/commits.py index f87e3587..84a6687e 100644 --- a/src/my/coding/commits.py +++ b/src/my/coding/commits.py @@ -206,7 +206,7 @@ def _cached_commits_path(p: Path) -> Path | str: # per-repo commits, to use cachew @mcachew( - depends_on=_repo_depends_on, + depends_on=_repo_depends_on, # ty: ignore[invalid-argument-type] # not sure why? possibly a bug logger=log, cache_path=_cached_commits_path, # type: ignore[arg-type] # hmm mypy seems confused here? likely a but in type + paramspec handling... ) diff --git a/src/my/core/_deprecated/dataset.py b/src/my/core/_deprecated/dataset.py index d7a427f9..ccb6bdca 100644 --- a/src/my/core/_deprecated/dataset.py +++ b/src/my/core/_deprecated/dataset.py @@ -9,4 +9,4 @@ def connect_readonly(db: PathIsh): # todo not sure if mode=ro has any benefit, but it doesn't work on read-only filesystems # maybe it should autodetect readonly filesystems and apply this? not sure creator = lambda: sqlite_connect_immutable(db) - return dataset.connect('sqlite:///', engine_kwargs={'creator': creator}) + return dataset.connect('sqlite:///', engine_kwargs={'creator': creator}) # ty: ignore[unresolved-attribute] diff --git a/src/my/core/cachew.py b/src/my/core/cachew.py index 4bcab7af..327b7347 100644 --- a/src/my/core/cachew.py +++ b/src/my/core/cachew.py @@ -132,11 +132,11 @@ def mcachew[F: Callable](fun: F) -> F: ... @overload def mcachew[F, **P]( - cache_path: PathProvider[P] | None = ..., # ty: ignore[too-many-positional-arguments] + cache_path: PathProvider[P] | None = ..., *, force_file: bool = ..., cls: type | None = ..., - depends_on: HashFunction[P] = ..., # ty: ignore[too-many-positional-arguments] + depends_on: HashFunction[P] = ..., logger: logging.Logger | None = ..., chunk_by: int = ..., synthetic_key: str | None = ..., diff --git a/src/my/core/cfg.py b/src/my/core/cfg.py index c62cdbcd..c407cae7 100644 --- a/src/my/core/cfg.py +++ b/src/my/core/cfg.py @@ -6,7 +6,7 @@ import sys from collections.abc import Callable, Iterator from contextlib import ExitStack, contextmanager -from typing import Any +from typing import Any, cast type Attrs = dict[str, Any] @@ -90,7 +90,8 @@ def tmp_config(*, modules: ModuleRegex | None = None, config=None): import my.config - with ExitStack() as module_reload_stack, _override_config(my.config) as new_config: + _config = cast(Any, my.config) # cast since ty doens't like module here (mypy infers ModuleType anyway) + with ExitStack() as module_reload_stack, _override_config(_config) as new_config: if config is not None: overrides = {k: v for k, v in vars(config).items() if not k.startswith('__')} for k, v in overrides.items(): diff --git a/src/my/core/common.py b/src/my/core/common.py index 0943e12b..0f693900 100644 --- a/src/my/core/common.py +++ b/src/my/core/common.py @@ -121,7 +121,7 @@ def prop(cls) -> str: return 'hello' res = C.prop - assert_type(res, str) # ty: ignore[type-assertion-failure] + assert_type(res, str) assert res == 'hello' diff --git a/src/my/core/core_config.py b/src/my/core/core_config.py index ea806b75..a53eb871 100644 --- a/src/my/core/core_config.py +++ b/src/my/core/core_config.py @@ -134,7 +134,7 @@ def matches(specs: Sequence[str]) -> str | None: def _reset_config() -> Iterator[Config]: # todo maybe have this decorator for the whole of my.config? from .cfg import _override_config - with _override_config(config) as cc: + with _override_config(config) as cc: # ty: ignore[invalid-argument-type] cc.enabled_modules = None cc.disabled_modules = None cc.cache_dir = None diff --git a/src/my/core/influxdb.py b/src/my/core/influxdb.py index fa82d4ce..067e669d 100644 --- a/src/my/core/influxdb.py +++ b/src/my/core/influxdb.py @@ -88,7 +88,6 @@ def dit() -> Iterable[Json]: 'fields': fields, } - # "The optimal batch size is 5000 lines of line protocol." # some chunking is def necessary, otherwise it fails inserted = 0 diff --git a/src/my/core/kompress.py b/src/my/core/kompress.py index 6fcf8047..e878af78 100644 --- a/src/my/core/kompress.py +++ b/src/my/core/kompress.py @@ -1,6 +1,8 @@ from . import warnings -warnings.high('my.core.kompress is deprecated. Install and use "kompress" library directly in your module (see https://github.com/karlicoss/kompress )') +warnings.high( + 'my.core.kompress is deprecated. Install and use "kompress" library directly in your module (see https://github.com/karlicoss/kompress )' +) from typing import TYPE_CHECKING diff --git a/src/my/core/orgmode.py b/src/my/core/orgmode.py index 09f4b846..667d7955 100644 --- a/src/my/core/orgmode.py +++ b/src/my/core/orgmode.py @@ -18,7 +18,7 @@ def parse_org_datetime(s: str) -> datetime: # todo not sure about these... fallback on 00:00? # ("%Y-%m-%d %a" , date), # ("%Y-%m-%d" , date), - ]: + ]: # fmt: skip try: return datetime.strptime(s, fmt) except ValueError: diff --git a/src/my/core/pandas.py b/src/my/core/pandas.py index bbc604b2..7ad7f407 100644 --- a/src/my/core/pandas.py +++ b/src/my/core/pandas.py @@ -28,10 +28,10 @@ if TYPE_CHECKING: import pandas as pd + from pandas._typing import S1 # meh type DataFrameT = pd.DataFrame - type SeriesT[T] = pd.Series[T] - from pandas._typing import S1 # meh + SeriesT = pd.Series # huh interesting -- with from __future__ import annotations don't even need else clause here? # but still if other modules import these we do need some fake runtime types here.. diff --git a/src/my/core/query.py b/src/my/core/query.py index 50b15b4b..03dd34d3 100644 --- a/src/my/core/query.py +++ b/src/my/core/query.py @@ -449,7 +449,7 @@ def select[U]( low(f"""Input was neither a function, or some iterable Expected 'src' to be an Iterable, but found {type(src).__name__}... Will attempt to call iter() on the value""") - it = src # ty: ignore[invalid-assignment] + it = src # try/catch an explicit iter() call to making this an Iterator, # to validate the input as something other helpers here can work with, diff --git a/src/my/core/stats.py b/src/my/core/stats.py index b39f8982..300c4df6 100644 --- a/src/my/core/stats.py +++ b/src/my/core/stats.py @@ -77,7 +77,7 @@ def stat( fname = func.__name__ # ty: ignore[unresolved-attribute] else: # meh. means it's just a list.. not sure how to generate a name then - fr = func # ty: ignore[invalid-assignment] + fr = func fname = f'unnamed_{id(fr)}' type_name = type(fr).__name__ extras = {} diff --git a/src/my/core/tests/test_get_files.py b/src/my/core/tests/test_get_files.py index 1548ee66..e2737090 100644 --- a/src/my/core/tests/test_get_files.py +++ b/src/my/core/tests/test_get_files.py @@ -151,6 +151,8 @@ def test_no_files(tmp_path_cwd: Path) -> None: def test_compressed(tmp_path_cwd: Path) -> None: + from typing import cast + file1 = tmp_path_cwd / 'file_1.zstd' file2 = tmp_path_cwd / 'file_2.zip' file3 = tmp_path_cwd / 'file_3.csv' @@ -167,7 +169,7 @@ def test_compressed(tmp_path_cwd: Path) -> None: assert not isinstance(res3, CPath) results = get_files( - [CPath(file1), ZipPath(file2), file3], + [CPath(file1), cast(Path, ZipPath(file2)), file3], # sorting a mixture of ZipPath/Path was broken in old kompress # it almost never happened though (usually it's only a bunch of ZipPath, so not a huge issue) sort=False, diff --git a/src/my/core/utils/itertools.py b/src/my/core/utils/itertools.py index f872d92f..af30e944 100644 --- a/src/my/core/utils/itertools.py +++ b/src/my/core/utils/itertools.py @@ -55,7 +55,7 @@ def make_dict[T, K, V]( *, key: Callable[[T], K], # TODO make value optional instead? but then will need a typing override for it? - value: Callable[[T], V] = _identity, # ty: ignore[invalid-parameter-default] + value: Callable[[T], V] = _identity, ) -> dict[K, V]: with_keys = ((key(i), i) for i in it) uniques = ensure_unique(with_keys, key=lambda p: p[0]) @@ -77,8 +77,9 @@ def test_make_dict() -> None: d = make_dict(it, key=lambda i: i % 2, value=lambda i: i) # check type inference - _d2: dict[str, int] = make_dict(it, key=lambda i: str(i)) - _d3: dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) + # TODO https://github.com/astral-sh/ty/issues/2095 + _d2: dict[str, int] = make_dict(it, key=lambda i: str(i)) # ty: ignore[invalid-assignment] + _d3: dict[str, bool] = make_dict(it, key=lambda i: str(i), value=lambda i: i % 2 == 0) # ty: ignore[invalid-assignment] @decorator @@ -110,7 +111,7 @@ def it() -> Iterator[int]: yield 2 res = it() - assert_type(res, list[int]) # ty: ignore[type-assertion-failure] + assert_type(res, list[int]) assert res == [1, 2] @@ -255,12 +256,12 @@ def test_check_if_hashable() -> None: x2: Iterator[int | str] = iter((123, 'aba')) r2 = check_if_hashable(x2) - assert_type(r2, Iterable[int | str]) # ty: ignore[type-assertion-failure] # atm ty is a bit confused about generics + assert_type(r2, Iterable[int | str]) assert list(r2) == [123, 'aba'] x3: tuple[object, ...] = (789, 'aba') r3 = check_if_hashable(x3) - assert_type(r3, Iterable[object]) # ty: ignore[type-assertion-failure] # ty thinks it's Literal[789, 'aba']? odd + assert_type(r3, Iterable[object]) assert r3 is x3 # object should be unchanged x4: list[set[int]] = [{1, 2, 3}, {4, 5, 6}] @@ -310,7 +311,7 @@ def unique_everseen[UET, UEU]( if callable(fun): iterable = fun() else: - iterable = fun # ty: ignore[invalid-assignment] # see https://github.com/astral-sh/ty/issues/117 + iterable = fun if key is None: # todo check key return type as well? but it's more likely to be hashable diff --git a/src/my/fbmessenger/__init__.py b/src/my/fbmessenger/__init__.py index 6c15e25a..88b9dd2c 100644 --- a/src/my/fbmessenger/__init__.py +++ b/src/my/fbmessenger/__init__.py @@ -30,4 +30,3 @@ if is_legacy_import: # todo not sure if possible to move this into legacy.py from .export import * - diff --git a/src/my/fbmessenger/all.py b/src/my/fbmessenger/all.py index a057dca8..c6b5d8e3 100644 --- a/src/my/fbmessenger/all.py +++ b/src/my/fbmessenger/all.py @@ -12,6 +12,7 @@ @src_export def _messages_export() -> Iterator[Res[Message]]: from . import export + # ok, this one is a little tricky # export.Message type is actually external (coming from fbmessengerexport module) # so it's unclear how to make mypy believe/check that common.Message is a structural subtype of export.Message @@ -32,6 +33,7 @@ def _messages_export() -> Iterator[Res[Message]]: @src_android def _messages_android() -> Iterator[Res[Message]]: from . import android + yield from android.messages() @@ -44,4 +46,5 @@ def messages() -> Iterator[Res[Message]]: def stats() -> Stats: from my.core import stat + return stat(messages) diff --git a/src/my/fbmessenger/common.py b/src/my/fbmessenger/common.py index b465bd5a..027ccfe1 100644 --- a/src/my/fbmessenger/common.py +++ b/src/my/fbmessenger/common.py @@ -58,6 +58,7 @@ def key(r: Res[Message]): # use both just in case, would be easier to spot tz issues # similar to twitter, might make sense to generify/document as a pattern return (r.id, r.dt) + yield from unique_everseen(chain(*sources), key=key) diff --git a/src/my/foursquare.py b/src/my/foursquare.py index 3b418aa3..13b7cdc5 100644 --- a/src/my/foursquare.py +++ b/src/my/foursquare.py @@ -25,7 +25,7 @@ def __init__(self, j) -> None: @property def summary(self) -> str: name = self.j.get('venue', {}).get('name', 'NO_NAME') - return "checked into " + name + " " + self.j.get('shout', "") # TODO should should be bold... + return "checked into " + name + " " + self.j.get('shout', "") # TODO should should be bold... @property def dt(self) -> datetime: @@ -56,6 +56,7 @@ def __init__(self, j) -> None: # TODO need json type + def get_raw(fname=None): if fname is None: fname = max(inputs()) @@ -90,6 +91,7 @@ def print_checkins(): def stats(): from more_itertools import ilen + return { 'checkins': ilen(get_checkins()), } diff --git a/src/my/github/all.py b/src/my/github/all.py index f5e13cf4..e86eb389 100644 --- a/src/my/github/all.py +++ b/src/my/github/all.py @@ -16,5 +16,6 @@ def events() -> Results: # todo hmm. not sure, maybe should be named sorted_events or something.. # also, not great that it's in all.py... think of a better way... def get_events() -> Results: - from ..core.error import sort_res_by + from my.core.error import sort_res_by + return sort_res_by(events(), key=lambda e: e.dt) diff --git a/src/my/github/common.py b/src/my/github/common.py index 50dac167..1960542a 100644 --- a/src/my/github/common.py +++ b/src/my/github/common.py @@ -16,6 +16,7 @@ logger = make_logger(__name__) + class Event(NamedTuple): dt: datetime summary: str @@ -27,9 +28,11 @@ class Event(NamedTuple): Results = Iterable[Res[Event]] + @warn_if_empty def merge_events(*sources: Results) -> Results: from itertools import chain + emitted: set[tuple[datetime, str]] = set() for e in chain(*sources): if isinstance(e, Exception): @@ -37,7 +40,7 @@ def merge_events(*sources: Results) -> Results: continue if e.is_bot: continue - key = (e.dt, e.eid) # use both just in case + key = (e.dt, e.eid) # use both just in case # TODO wtf?? some minor (e.g. 1 sec) discrepancies (e.g. create repository events) if key in emitted: logger.debug('ignoring %s: %s', key, e) diff --git a/src/my/goodreads.py b/src/my/goodreads.py index 3b295381..1783d034 100644 --- a/src/my/goodreads.py +++ b/src/my/goodreads.py @@ -1,6 +1,7 @@ """ [[https://www.goodreads.com][Goodreads]] statistics """ + REQUIRES = [ 'goodrexport @ git+https://github.com/karlicoss/goodrexport', ] @@ -17,16 +18,20 @@ class goodreads(user_config): # paths[s]/glob to the exported JSON data export_path: Paths + from my.core.cfg import Attrs, make_config def _migration(attrs: Attrs) -> Attrs: export_dir = 'export_dir' - if export_dir in attrs: # legacy name + if export_dir in attrs: # legacy name attrs['export_path'] = attrs[export_dir] from my.core.warnings import high + high(f'"{export_dir}" is deprecated! Please use "export_path" instead."') return attrs + + config = make_config(goodreads, migration=_migration) #############################3 @@ -65,6 +70,7 @@ def books() -> Iterator[dal.Book]: ####### # todo ok, not sure these really belong here... + @dataclass class Event: dt: datetime_aware @@ -76,8 +82,8 @@ def events() -> Iterator[Event]: for b in books(): yield Event( dt=b.date_added, - summary=f'Added book "{b.title}"', # todo shelf? - eid=b.id + summary=f'Added book "{b.title}"', # todo shelf? + eid=b.id, ) # todo finished? other updates? @@ -97,6 +103,7 @@ def fmtdt(dt): return dt tz = pytz.timezone('Europe/London') return dt.astimezone(tz) + for b in sorted(books(), key=key): print(f""" {b.title} by {', '.join(b.authors)} diff --git a/src/my/hypothesis.py b/src/my/hypothesis.py index 2d1519d0..b0a9acfb 100644 --- a/src/my/hypothesis.py +++ b/src/my/hypothesis.py @@ -1,6 +1,7 @@ """ [[https://hypothes.is][Hypothes.is]] highlights and annotations """ + REQUIRES = [ 'hypexport @ git+https://github.com/karlicoss/hypexport', ] diff --git a/src/my/instagram/all.py b/src/my/instagram/all.py index ce78409a..0925f251 100644 --- a/src/my/instagram/all.py +++ b/src/my/instagram/all.py @@ -6,16 +6,22 @@ from .common import Message, _merge_messages src_gdpr = import_source(module_name='my.instagram.gdpr') + + @src_gdpr def _messages_gdpr() -> Iterator[Res[Message]]: from . import gdpr + yield from gdpr.messages() src_android = import_source(module_name='my.instagram.android') + + @src_android def _messages_android() -> Iterator[Res[Message]]: from . import android + yield from android.messages() diff --git a/src/my/ip/all.py b/src/my/ip/all.py index 954f6200..9b4108a6 100644 --- a/src/my/ip/all.py +++ b/src/my/ip/all.py @@ -6,9 +6,7 @@ For an example of how this could be used, see https://github.com/purarue/HPI/tree/master/my/ip """ -REQUIRES = [ - "ipgeocache @ git+https://github.com/purarue/ipgeocache" -] +REQUIRES = ["ipgeocache @ git+https://github.com/purarue/ipgeocache"] from collections.abc import Iterator diff --git a/src/my/jawbone/__init__.py b/src/my/jawbone/__init__.py index 4ec2fbee..0381e527 100644 --- a/src/my/jawbone/__init__.py +++ b/src/my/jawbone/__init__.py @@ -21,14 +21,16 @@ GRAPHS_DIR = BDIR / 'graphs' - -XID = str # TODO how to shared with backup thing? +XID = str # TODO how to shared with backup thing? Phases = dict[XID, Any] + + @lru_cache(1) def get_phases() -> Phases: return json.loads(PHASES_FILE.read_text()) + # TODO use awakenings and quality class SleepEntry: def __init__(self, js) -> None: @@ -73,7 +75,7 @@ def asleep(self) -> datetime: @property def sleep_start(self) -> datetime: - return self.asleep # TODO careful, maybe use same logic as emfit + return self.asleep # TODO careful, maybe use same logic as emfit @property def bed_time(self) -> int: @@ -113,7 +115,7 @@ def pre_dataframe() -> Iterable[Res[SleepEntry]]: sleeps = load_sleeps() # todo emit error if graph doesn't exist?? - sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. + sleeps = [s for s in sleeps if s.graph.exists()] # TODO careful.. bucketed = bucket(sleeps, key=lambda s: s.date_) @@ -136,7 +138,7 @@ def dataframe(): if isinstance(s, Exception): dt = extract_error_datetime(s) d = { - 'date' : dt, + 'date': dt, 'error': str(s), } else: @@ -152,12 +154,14 @@ def dataframe(): dicts.append(d) import pandas as pd + return pd.DataFrame(dicts) # TODO tz is in sleeps json def stats(): from ..core import stat + return stat(pre_dataframe) @@ -221,7 +225,7 @@ def plot_one(sleep: SleepEntry, fig, axes, xlims=None, *, showtext=True): length=0, labelsize=7, rotation=30, - pad=-14, # err... hacky + pad=-14, # err... hacky ) ylims = [0, 50] @@ -248,9 +252,10 @@ def plot_one(sleep: SleepEntry, fig, axes, xlims=None, *, showtext=True): # import melatonin # dt = melatonin.get_data() + def predicate(sleep: SleepEntry): """ - Filter for comparing similar sleep sessions + Filter for comparing similar sleep sessions """ start = sleep.created.time() end = sleep.completed.time() @@ -265,7 +270,7 @@ def plot() -> None: from matplotlib.figure import Figure # type: ignore[import-not-found] # TODO FIXME melatonin data - melatonin_data = {} # type: ignore[var-annotated] + melatonin_data = {} # type: ignore[var-annotated] # TODO ?? sleeps = list(filter(predicate, load_sleeps())) @@ -275,7 +280,7 @@ def plot() -> None: fig: Figure = plt.figure(figsize=(15, sleeps_count * 1)) axarr = fig.subplots(nrows=len(sleeps)) - for (sleep, axes) in zip(sleeps, axarr, strict=True): + for sleep, axes in zip(sleeps, axarr, strict=True): plot_one(sleep, fig, axes, showtext=True) used = melatonin_data.get(sleep.date_, None) sused: str @@ -294,11 +299,9 @@ def plot() -> None: axes.patch.set_alpha(0.5) axes.set_facecolor(color) - plt.tight_layout() plt.subplots_adjust(hspace=0.0) # er... this saves with a different aspect ratio for some reason. # tap 'ctrl-s' on mpl plot window to save.. # plt.savefig('res.png', asp) plt.show() - diff --git a/tox.ini b/tox.ini index 53edc419..fbd458ab 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] -minversion = 3.21 +minversion = 4 + # relies on the correct version of Python installed +# (we rely on CI for the test matrix) envlist = ruff,tests-core,tests-all,demo,mypy-core,ty-core,mypy-all,ty-all # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory @@ -23,8 +25,8 @@ set_env = # generally this is more robust and safer, prevents weird issues later on PYTHONSAFEPATH=1 -# default is 'editable', in which tox builds wheel first for some reason? not sure if makes much sense -package = uv-editable +runner = uv-venv-lock-runner +uv_sync_locked = false [testenv:ruff] @@ -84,21 +86,11 @@ commands = [testenv:demo] set_env = -# ugh. the demo test relies on 'current' directory path, so need to undy the PYTHONSAFEPATH set above +# ugh. the demo test relies on 'current' directory path, so need to undo the PYTHONSAFEPATH set above # the whole demo test is a bit crap, should really migrate to something more robust PYTHONSAFEPATH= # another issue is that it's installing HPI, and then patching/ trying to use the 'local' version -- really not ideal.. skip_install = true -deps = - git+https://github.com/karlicoss/hypexport - # copy the dependencies from pyproject.toml for now - pytz - typing-extensions - platformdirs - more-itertools - decorator - click - kompress commands = {envpython} ./demo.py