Add importers for various services and refactor base importer class

pyproject.toml

@@ -106,9 +106,18 @@ flake8 = "^7.3.0"
 
 
 [tool.poetry]
+packages = [{ include = "lifestream", from = "src" }]
 
-package-mode = false
-
+[tool.poetry.scripts]
+lifestream-import = "lifestream.cli:main"
+lifestream-lastfm = "lifestream.importers.lastfm:main"
+lifestream-atom = "lifestream.importers.atom:main"
+lifestream-github = "lifestream.importers.github_commits:main"
+lifestream-flickr = "lifestream.importers.flickr:main"
+lifestream-mastodon = "lifestream.importers.mastodon_toots:main"
+lifestream-steam = "lifestream.importers.steam:main"
+lifestream-switchbot = "lifestream.importers.switchbot:main"
+lifestream-wordpress = "lifestream.importers.wordpress:main"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

pytest.ini

@@ -5,6 +5,7 @@ testpaths = tests contrib/ffxiv-upload-achievement-icons/tests
 python_files = test_*.py
 python_classes = Test*
 python_functions = test_*
+pythonpath = src
 addopts = -v --tb=short
 filterwarnings =
     ignore::DeprecationWarning

scheduler.py

@@ -17,7 +17,6 @@
 
 import argparse
 import logging
-import os
 import signal
 import sys
 from datetime import datetime
@@ -42,14 +41,9 @@
 from apscheduler.schedulers.blocking import BlockingScheduler  # noqa: E402
 from apscheduler.triggers.cron import CronTrigger  # noqa: E402
 
-# Set up path for imports - lifestream expects to run from imports/ directory
-basedir = os.path.dirname(os.path.abspath(__file__))
-imports_dir = os.path.join(basedir, "imports")
-os.chdir(imports_dir)  # Change to imports/ so lifestream finds config correctly
-sys.path.insert(0, imports_dir)
-
-import lifestream  # noqa: E402
-from lifestream.jobs import run_import, run_shell_command  # noqa: E402
+# Import from the new package structure
+from lifestream.core import config  # noqa: E402
+from lifestream.core.jobs import run_import, run_shell_command  # noqa: E402
 
 logger = logging.getLogger("Scheduler")
 
@@ -64,11 +58,11 @@ def get_schedules():
     """Read schedules from config.ini [schedules] section."""
     schedules = {}
 
-    if not lifestream.config.has_section("schedules"):
+    if not config.has_section("schedules"):
         logger.warning("No [schedules] section found in config.ini")
         return schedules
 
-    for job_name, cron_expr in lifestream.config.items("schedules"):
+    for job_name, cron_expr in config.items("schedules"):
         # Skip disabled jobs (commented with ; or #, or empty value)
         if not cron_expr or cron_expr.startswith(";") or cron_expr.startswith("#"):
             continue
@@ -100,10 +94,10 @@ def get_schedules():
 def create_scheduler():
     """Create and configure the APScheduler instance."""
     # Get Redis connection settings from config
-    redis_host = lifestream.config.get("redis", "host", fallback="localhost")
-    redis_port = int(lifestream.config.get("redis", "port", fallback=6379))
-    redis_username = lifestream.config.get("redis", "username", fallback=None)
-    redis_password = lifestream.config.get("redis", "password", fallback=None)
+    redis_host = config.get("redis", "host", fallback="localhost")
+    redis_port = int(config.get("redis", "port", fallback=6379))
+    redis_username = config.get("redis", "username", fallback=None)
+    redis_password = config.get("redis", "password", fallback=None)
 
     jobstores = {
         "default": RedisJobStore(
@@ -140,8 +134,8 @@ def add_jobs(scheduler):
     """Add all configured jobs to the scheduler."""
     schedules = get_schedules()
 
-    for job_name, config in schedules.items():
-        cron = config["cron"]
+    for job_name, job_config in schedules.items():
+        cron = job_config["cron"]
 
         try:
             trigger = CronTrigger.from_crontab(cron)
@@ -159,8 +153,8 @@ def add_jobs(scheduler):
                 args=[actual_name, command],
                 id=actual_name,
                 name=actual_name,
-                misfire_grace_time=config["misfire_grace_time"],
-                coalesce=config["coalesce"],
+                misfire_grace_time=job_config["misfire_grace_time"],
+                coalesce=job_config["coalesce"],
                 replace_existing=True,
             )
         else:
@@ -170,8 +164,8 @@ def add_jobs(scheduler):
                 args=[job_name],
                 id=job_name,
                 name=job_name,
-                misfire_grace_time=config["misfire_grace_time"],
-                coalesce=config["coalesce"],
+                misfire_grace_time=job_config["misfire_grace_time"],
+                coalesce=job_config["coalesce"],
                 replace_existing=True,
             )
 
@@ -191,9 +185,9 @@ def list_jobs():
     print(f"{'Job Name':<25} {'Schedule':<20} {'Grace(s)':<10} {'Coalesce'}")
     print("-" * 70)
 
-    for job_name, config in sorted(schedules.items()):
+    for job_name, job_config in sorted(schedules.items()):
         print(
-            f"{job_name:<25} {config['cron']:<20} {config['misfire_grace_time']:<10} {config['coalesce']}"
+            f"{job_name:<25} {job_config['cron']:<20} {job_config['misfire_grace_time']:<10} {job_config['coalesce']}"
         )
 
 

src/lifestream/__init__.py

@@ -0,0 +1,56 @@
+"""
+Lifestream - A personal data aggregation system.
+
+This package provides functionality for importing personal data from various
+online services into a unified database.
+"""
+
+__version__ = "2.0.0"
+
+# Re-export commonly used items for convenience
+from lifestream.core.config import (
+    config,
+    get_secrets_dir,
+    get_log_dir,
+    get_project_root,
+)
+from lifestream.core.logging import setup_logging, get_logger
+from lifestream.core.db import EntryStore, get_connection, get_cursor
+from lifestream.core.cache import get_redis_connection, file_cache
+from lifestream.core.utils import (
+    niceTimeDelta,
+    yearsago,
+    force_json,
+    is_jsonable,
+    convertNiceTime,
+    AnAttributeError,
+)
+__all__ = [
+    # Config
+    "config",
+    "get_secrets_dir",
+    "get_log_dir",
+    "get_project_root",
+    # Logging
+    "setup_logging",
+    "get_logger",
+    # Database
+    "EntryStore",
+    "get_connection",
+    "get_cursor",
+    # Cache
+    "get_redis_connection",
+    "file_cache",
+    # Utils
+    "niceTimeDelta",
+    "yearsago",
+    "force_json",
+    "is_jsonable",
+    "convertNiceTime",
+    "AnAttributeError",
+    # OAuth
+    "read_token_file",
+    "write_token_file",
+    # Notifications
+    "send_failure_notifications",
+]

src/lifestream/cli.py

@@ -0,0 +1,63 @@
+"""
+Command-line interface for Lifestream importers.
+
+Run a specific importer:
+    lifestream-import <importer_name> [options]
+
+List available importers:
+    lifestream-import --list
+"""
+
+import argparse
+import sys
+
+from lifestream.importers import IMPORTERS
+
+
+def main():
+    """Main entry point for the lifestream-import command."""
+    parser = argparse.ArgumentParser(
+        description="Run Lifestream importers",
+        prog="lifestream-import",
+    )
+
+    parser.add_argument(
+        "--list",
+        "-l",
+        action="store_true",
+        help="List available importers",
+    )
+
+    parser.add_argument(
+        "importer",
+        nargs="?",
+        help="Name of the importer to run",
+    )
+
+    # Parse known args only, pass rest to importer
+    args, remaining = parser.parse_known_args()
+
+    if args.list:
+        print("Available importers:")
+        for name, cls in sorted(IMPORTERS.items()):
+            print(f"  {name:15} - {cls.description}")
+        return 0
+
+    if not args.importer:
+        parser.print_help()
+        print("\nUse --list to see available importers")
+        return 1
+
+    importer_name = args.importer.lower()
+
+    if importer_name not in IMPORTERS:
+        print(f"Unknown importer: {importer_name}")
+        print("Use --list to see available importers")
+        return 1
+
+    importer_cls = IMPORTERS[importer_name]
+    return importer_cls.main(remaining)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/lifestream/core/__init__.py

@@ -0,0 +1,84 @@
+"""
+Core functionality for Lifestream.
+
+This module provides:
+- Configuration management
+- Database access
+- Caching with Redis
+- Logging setup
+- Utility functions
+"""
+
+from .config import (
+    config,
+    get_project_root,
+    get_secrets_dir,
+    get_log_dir,
+    resolve_path,
+    get_config_value,
+    get_config_bool,
+)
+from .logging import setup_logging, get_logger, is_logging_configured
+from .db import EntryStore, get_connection, get_cursor, set_no_db_mode, get_no_db_mode
+from .cache import (
+    get_redis_connection,
+    set_backoff,
+    should_backoff,
+    check_and_set_backoff,
+    file_cache,
+)
+from .utils import (
+    niceTimeDelta,
+    yearsago,
+    force_json,
+    is_jsonable,
+    convertNiceTime,
+    AnAttributeError,
+)
+from .oauth_utils import read_token_file, write_token_file
+from .notifications import (
+    send_failure_notifications,
+    send_failure_email,
+    send_failure_slack,
+)
+
+__all__ = [
+    # Config
+    "config",
+    "get_project_root",
+    "get_secrets_dir",
+    "get_log_dir",
+    "resolve_path",
+    "get_config_value",
+    "get_config_bool",
+    # Logging
+    "setup_logging",
+    "get_logger",
+    "is_logging_configured",
+    # Database
+    "EntryStore",
+    "get_connection",
+    "get_cursor",
+    "set_no_db_mode",
+    "get_no_db_mode",
+    # Cache
+    "get_redis_connection",
+    "set_backoff",
+    "should_backoff",
+    "check_and_set_backoff",
+    "file_cache",
+    # Utils
+    "niceTimeDelta",
+    "yearsago",
+    "force_json",
+    "is_jsonable",
+    "convertNiceTime",
+    "AnAttributeError",
+    # OAuth
+    "read_token_file",
+    "write_token_file",
+    # Notifications
+    "send_failure_notifications",
+    "send_failure_email",
+    "send_failure_slack",
+]