From 98d567581cd89b3d4d20687c010d3905ae4983fa Mon Sep 17 00:00:00 2001 From: Serhii Ponomarov Date: Wed, 28 Jan 2026 18:33:00 +0200 Subject: [PATCH] test(live): add sandboxed live test suite --- Makefile | 3 + docs/DEVELOPER_GUIDE.md | 13 +++ pyproject.toml | 14 ++- tests/conftest.py | 11 +- tests/live/README.md | 26 +++++ tests/live/__init__.py | 1 + tests/live/conftest.py | 29 +++++ tests/live/helpers.py | 170 ++++++++++++++++++++++++++++ tests/live/test_connections_live.py | 58 ++++++++++ tests/live/test_connectors_live.py | 25 ++++ tests/live/test_profiles_live.py | 41 +++++++ tests/live/test_projects_live.py | 104 +++++++++++++++++ tests/live/test_recipes_live.py | 89 +++++++++++++++ tests/live/test_workspace_live.py | 24 ++++ 14 files changed, 606 insertions(+), 2 deletions(-) create mode 100644 tests/live/README.md create mode 100644 tests/live/__init__.py create mode 100644 tests/live/conftest.py create mode 100644 tests/live/helpers.py create mode 100644 tests/live/test_connections_live.py create mode 100644 tests/live/test_connectors_live.py create mode 100644 tests/live/test_profiles_live.py create mode 100644 tests/live/test_projects_live.py create mode 100644 tests/live/test_recipes_live.py create mode 100644 tests/live/test_workspace_live.py diff --git a/Makefile b/Makefile index 53dae42..4fe1033 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,9 @@ test-unit: test-integration: uv run pytest tests/integration/ -v +test-live: + uv run pytest -m live -vv -s + test-client: uv run pytest src/workato_platform_cli/client/workato_api/test/ -v diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index adeb80e..a1cb687 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -104,6 +104,19 @@ If you want to contribute to the Workato CLI codebase itself, use these developm make test # Run tests and check code style ``` +### Live Tests (Sandbox Only) +Live tests hit real Workato APIs and require a sandbox or trial workspace. + +```bash +WORKATO_LIVE_SANDBOX=1 make test-live +``` + +Required environment variables: +- `WORKATO_HOST` +- `WORKATO_API_TOKEN` + +Optional environment variables and flags are documented in `tests/live/README.md`. + ### Testing with Test PyPI When testing pre-release versions from Test PyPI, you need to use both Test PyPI and regular PyPI to resolve dependencies: diff --git a/pyproject.toml b/pyproject.toml index f523ca0..4639c4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,7 +197,19 @@ ignore_missing_imports = true # Pytest configuration [tool.pytest.ini_options] minversion = "7.0" -addopts = "-ra -q" + +# Default pytest flags: +# -ra: show extra summary for skipped/failed/etc +# -q: quiet +# -m "not live": never run real sandbox tests unless explicitly requested +addopts = '-ra -q -m "not live"' + +# Markers used across the suite +markers = [ + "live: hits real sandbox endpoints (skipped by default; requires env vars like WORKATO_HOST/WORKATO_API_TOKEN)", + "stubbed: uses mocked/stubbed HTTP (default tier for integration workflow tests)", +] + testpaths = ["tests"] python_files = ["test_*.py", "*_test.py"] python_classes = ["Test*"] diff --git a/tests/conftest.py b/tests/conftest.py index 4530a87..c1d1ce3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,11 +50,20 @@ def mock_workato_client() -> Mock: @pytest.fixture(autouse=True) -def isolate_tests(monkeypatch: pytest.MonkeyPatch, temp_config_dir: Path) -> None: +def isolate_tests( + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + temp_config_dir: Path, +) -> None: """Isolate tests by using temporary directories and env vars.""" # Prevent tests from accessing real config files monkeypatch.setenv("WORKATO_CONFIG_DIR", str(temp_config_dir)) monkeypatch.setenv("WORKATO_DISABLE_UPDATE_CHECK", "1") + monkeypatch.setattr("pathlib.Path.home", lambda: temp_config_dir) + + # If this is a live test, do NOT force test mode and do NOT wipe creds + if request.node.get_closest_marker("live"): + return # Ensure we don't make real API calls monkeypatch.setenv("WORKATO_TEST_MODE", "1") diff --git a/tests/live/README.md b/tests/live/README.md new file mode 100644 index 0000000..2929d3c --- /dev/null +++ b/tests/live/README.md @@ -0,0 +1,26 @@ +# Live Tests (Sandbox Only) + +These tests hit real Workato APIs. Run only against sandbox/trial environments. + +## Required Environment Variables +- `WORKATO_HOST` +- `WORKATO_API_TOKEN` +- `WORKATO_LIVE_SANDBOX=1` (explicit confirmation) + +## Optional Environment Variables +- `WORKATO_TEST_PROJECT_ID` or `WORKATO_TEST_PROJECT_NAME` +- `WORKATO_TEST_PROFILE` (default: `live-test`) +- `WORKATO_LIVE_ALLOW_PUSH=1` (enable `workato push`) +- `WORKATO_TEST_RECIPE_ID` (enable recipe start/stop) +- `WORKATO_LIVE_ALLOW_RECIPE_CONTROL=1` (enable recipe start/stop) +- `WORKATO_TEST_CONNECTION_ID` (enable OAuth URL and pick-list tests) +- `WORKATO_TEST_PICKLIST_NAME` (enable pick-list test) + +## Run +```bash +WORKATO_LIVE_SANDBOX=1 make test-live +``` + +## Safety Notes +- Destructive actions are opt-in. +- Project creation test skips if the token lacks permissions. diff --git a/tests/live/__init__.py b/tests/live/__init__.py new file mode 100644 index 0000000..de3afc9 --- /dev/null +++ b/tests/live/__init__.py @@ -0,0 +1 @@ +"""Live test package marker.""" diff --git a/tests/live/conftest.py b/tests/live/conftest.py new file mode 100644 index 0000000..c2ee228 --- /dev/null +++ b/tests/live/conftest.py @@ -0,0 +1,29 @@ +import contextlib + +from collections.abc import AsyncGenerator +from typing import Any + +import pytest +import pytest_asyncio + +from workato_platform_cli import Workato + + +@pytest_asyncio.fixture(autouse=True) +async def close_workato_clients( + monkeypatch: pytest.MonkeyPatch, +) -> AsyncGenerator[None, None]: + clients: list[Workato] = [] + original_init = Workato.__init__ + + def tracking_init(self: Workato, *args: Any, **kwargs: Any) -> None: + original_init(self, *args, **kwargs) + clients.append(self) + + monkeypatch.setattr(Workato, "__init__", tracking_init) + + yield + + for client in clients: + with contextlib.suppress(Exception): + await client.close() diff --git a/tests/live/helpers.py b/tests/live/helpers.py new file mode 100644 index 0000000..d2435a2 --- /dev/null +++ b/tests/live/helpers.py @@ -0,0 +1,170 @@ +import json +import os + +from pathlib import Path +from typing import cast + +import certifi +import pytest + +from asyncclick.testing import CliRunner + +from workato_platform_cli import Workato +from workato_platform_cli.cli import cli +from workato_platform_cli.cli.commands.projects.project_manager import ProjectManager +from workato_platform_cli.client.workato_api.configuration import Configuration + + +REQUIRED_BASE = ["WORKATO_HOST", "WORKATO_API_TOKEN"] +_PROJECT_ID_CACHE: str | None = None + + +def ensure_sandbox() -> None: + host = os.getenv("WORKATO_HOST", "").lower() + if os.getenv("WORKATO_LIVE_SANDBOX", "").lower() in {"1", "true", "yes"}: + return + if any(token in host for token in ("trial", "preview", "sandbox")): + return + pytest.skip( + "Set WORKATO_LIVE_SANDBOX=1 to confirm tests are running against a sandbox." + ) + + +def require_live_env(extra: list[str] | None = None) -> None: + ensure_sandbox() + required = REQUIRED_BASE + (extra or []) + missing = [key for key in required if not os.getenv(key)] + if missing: + pytest.skip(f"Missing env vars for live tests: {', '.join(missing)}") + + +def allow_live_action(flag: str) -> bool: + return os.getenv(flag, "").lower() in {"1", "true", "yes"} + + +def force_keyring_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + import workato_platform_cli.cli.utils.config.profiles as profiles + + def _raise_no_keyring() -> None: + raise Exception("No keyring backend") + + monkeypatch.setattr(profiles.keyring, "get_keyring", _raise_no_keyring) + + +def prepare_live_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("WORKATO_DISABLE_UPDATE_CHECK", "1") + monkeypatch.setattr(Path, "home", lambda: tmp_path) + force_keyring_fallback(monkeypatch) + + +def _build_api_client() -> Workato: + api_host = os.environ["WORKATO_HOST"] + api_token = os.environ["WORKATO_API_TOKEN"] + api_config = Configuration( + access_token=api_token, + host=api_host, + ssl_ca_cert=certifi.where(), + ) + return Workato(configuration=api_config) + + +async def resolve_project_id() -> str: + global _PROJECT_ID_CACHE + if _PROJECT_ID_CACHE: + return _PROJECT_ID_CACHE + + project_id = os.getenv("WORKATO_TEST_PROJECT_ID") + if project_id: + _PROJECT_ID_CACHE = project_id + return project_id + + project_name = os.getenv("WORKATO_TEST_PROJECT_NAME") + + async with _build_api_client() as workato_api_client: + project_manager = ProjectManager(workato_api_client=workato_api_client) + projects = await project_manager.get_all_projects() + + if not projects: + pytest.skip("No projects found in workspace.") + + if project_name: + for project in projects: + if project.name == project_name: + _PROJECT_ID_CACHE = str(project.id) + return str(project.id) + pytest.skip(f"No project found with name '{project_name}'.") + + _PROJECT_ID_CACHE = str(projects[0].id) + return _PROJECT_ID_CACHE + + +async def delete_project(project_id: int) -> None: + async with _build_api_client() as workato_api_client: + await workato_api_client.projects_api.delete_project(project_id=project_id) + + +async def run_init(cli_runner: CliRunner, project_id: str) -> dict[str, object]: + api_host = os.environ["WORKATO_HOST"] + profile_name = os.getenv("WORKATO_TEST_PROFILE", "live-test") + result = await cli_runner.invoke( + cli, + [ + "init", + "--non-interactive", + "--profile", + profile_name, + "--project-id", + str(project_id), + "--output-mode", + "json", + "--region", + "custom", + "--api-url", + api_host, + ], + ) + + assert result.exit_code == 0, result.output + + lines = [line for line in result.output.splitlines() if line.strip()] + data = cast(dict[str, object], json.loads(lines[-1])) + assert data.get("status") == "success" + return data + + +async def run_init_create_project( + cli_runner: CliRunner, + project_name: str, +) -> dict[str, object]: + api_host = os.environ["WORKATO_HOST"] + profile_name = os.getenv("WORKATO_TEST_PROFILE", "live-test") + result = await cli_runner.invoke( + cli, + [ + "init", + "--non-interactive", + "--profile", + profile_name, + "--project-name", + project_name, + "--output-mode", + "json", + "--region", + "custom", + "--api-url", + api_host, + ], + ) + + if result.exit_code != 0: + if "UNAUTHORIZED" in result.output or "Authentication failed" in result.output: + pytest.skip( + "Project creation/pull not authorized for this token. " + "Check sandbox permissions or token scope." + ) + assert result.exit_code == 0, result.output + + lines = [line for line in result.output.splitlines() if line.strip()] + data = cast(dict[str, object], json.loads(lines[-1])) + assert data.get("status") == "success" + return data diff --git a/tests/live/test_connections_live.py b/tests/live/test_connections_live.py new file mode 100644 index 0000000..197bef7 --- /dev/null +++ b/tests/live/test_connections_live.py @@ -0,0 +1,58 @@ +import os + +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import prepare_live_env, require_live_env +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_connections_oauth_flow_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env(["WORKATO_TEST_CONNECTION_ID"]) + prepare_live_env(monkeypatch, tmp_path) + + connection_id = os.environ["WORKATO_TEST_CONNECTION_ID"] + + list_result = await cli_runner.invoke(cli, ["connections", "list"]) + assert list_result.exit_code == 0, list_result.output + + oauth_result = await cli_runner.invoke( + cli, ["connections", "get-oauth-url", "--id", str(connection_id)] + ) + assert oauth_result.exit_code == 0, oauth_result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_connection_picklist_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env(["WORKATO_TEST_CONNECTION_ID", "WORKATO_TEST_PICKLIST_NAME"]) + prepare_live_env(monkeypatch, tmp_path) + + connection_id = os.environ["WORKATO_TEST_CONNECTION_ID"] + picklist_name = os.environ["WORKATO_TEST_PICKLIST_NAME"] + + result = await cli_runner.invoke( + cli, + [ + "connections", + "pick-list", + "--id", + str(connection_id), + "--pick-list-name", + picklist_name, + ], + ) + assert result.exit_code == 0, result.output diff --git a/tests/live/test_connectors_live.py b/tests/live/test_connectors_live.py new file mode 100644 index 0000000..d51aaf8 --- /dev/null +++ b/tests/live/test_connectors_live.py @@ -0,0 +1,25 @@ +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import prepare_live_env, require_live_env +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_connectors_discovery_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + list_result = await cli_runner.invoke(cli, ["connectors", "list"]) + assert list_result.exit_code == 0, list_result.output + + params_result = await cli_runner.invoke(cli, ["connectors", "parameters"]) + assert params_result.exit_code == 0, params_result.output diff --git a/tests/live/test_profiles_live.py b/tests/live/test_profiles_live.py new file mode 100644 index 0000000..f7ff839 --- /dev/null +++ b/tests/live/test_profiles_live.py @@ -0,0 +1,41 @@ +import os + +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import ( + prepare_live_env, + require_live_env, + resolve_project_id, + run_init, +) +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_profiles_workflow_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + project_id = await resolve_project_id() + + with cli_runner.isolated_filesystem(): + await run_init(cli_runner, project_id) + + list_result = await cli_runner.invoke(cli, ["profiles", "list"]) + assert list_result.exit_code == 0, list_result.output + + status_result = await cli_runner.invoke(cli, ["profiles", "status"]) + assert status_result.exit_code == 0, status_result.output + + profile_name = os.getenv("WORKATO_TEST_PROFILE", "live-test") + use_result = await cli_runner.invoke(cli, ["profiles", "use", profile_name]) + assert use_result.exit_code == 0, use_result.output diff --git a/tests/live/test_projects_live.py b/tests/live/test_projects_live.py new file mode 100644 index 0000000..3a68ac4 --- /dev/null +++ b/tests/live/test_projects_live.py @@ -0,0 +1,104 @@ +import time + +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import ( + allow_live_action, + delete_project, + prepare_live_env, + require_live_env, + resolve_project_id, + run_init, + run_init_create_project, +) +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_project_bootstrap_sync_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + project_id = await resolve_project_id() + + with cli_runner.isolated_filesystem(): + await run_init(cli_runner, project_id) + + pull_result = await cli_runner.invoke(cli, ["pull"]) + assert pull_result.exit_code == 0, pull_result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_project_push_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + if not allow_live_action("WORKATO_LIVE_ALLOW_PUSH"): + pytest.skip("Set WORKATO_LIVE_ALLOW_PUSH=1 to run live push") + + prepare_live_env(monkeypatch, tmp_path) + + project_id = await resolve_project_id() + + with cli_runner.isolated_filesystem(): + await run_init(cli_runner, project_id) + + push_result = await cli_runner.invoke(cli, ["push"]) + assert push_result.exit_code == 0, push_result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_projects_list_remote_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + result = await cli_runner.invoke( + cli, ["projects", "list", "--source", "remote", "--output-mode", "json"] + ) + assert result.exit_code == 0, result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_project_create_and_cleanup_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + project_prefix = "cli-live" + project_name = f"{project_prefix}-{int(time.time())}" + + with cli_runner.isolated_filesystem(): + result = await run_init_create_project(cli_runner, project_name) + project_info = result.get("project") + project_id = None + if isinstance(project_info, dict): + project_id = project_info.get("id") + if not project_id: + pytest.skip("Project ID not returned from init; cannot clean up.") + + try: + list_result = await cli_runner.invoke(cli, ["projects", "list"]) + assert list_result.exit_code == 0, list_result.output + finally: + await delete_project(int(project_id)) diff --git a/tests/live/test_recipes_live.py b/tests/live/test_recipes_live.py new file mode 100644 index 0000000..499c9c2 --- /dev/null +++ b/tests/live/test_recipes_live.py @@ -0,0 +1,89 @@ +import os + +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import ( + allow_live_action, + prepare_live_env, + require_live_env, + resolve_project_id, + run_init, +) +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_recipes_list_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + project_id = await resolve_project_id() + + with cli_runner.isolated_filesystem(): + await run_init(cli_runner, project_id) + + list_result = await cli_runner.invoke(cli, ["recipes", "list"]) + assert list_result.exit_code == 0, list_result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_recipes_start_stop_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env(["WORKATO_TEST_RECIPE_ID"]) + if not allow_live_action("WORKATO_LIVE_ALLOW_RECIPE_CONTROL"): + pytest.skip("Set WORKATO_LIVE_ALLOW_RECIPE_CONTROL=1 to run start/stop") + + prepare_live_env(monkeypatch, tmp_path) + + project_id = await resolve_project_id() + recipe_id = os.environ["WORKATO_TEST_RECIPE_ID"] + + with cli_runner.isolated_filesystem(): + await run_init(cli_runner, project_id) + + start_result = await cli_runner.invoke( + cli, ["recipes", "start", "--id", str(recipe_id)] + ) + assert start_result.exit_code == 0, start_result.output + + stop_result = await cli_runner.invoke( + cli, ["recipes", "stop", "--id", str(recipe_id)] + ) + assert stop_result.exit_code == 0, stop_result.output + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_recipe_validate_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + fixtures_dir = Path(__file__).resolve().parents[1] / "fixtures" + sample_recipe = fixtures_dir / "sample_recipe.json" + + with cli_runner.isolated_filesystem(): + recipe_path = Path("sample_recipe.json") + recipe_contents = sample_recipe.read_text(encoding="utf-8") + recipe_path.write_text(recipe_contents, encoding="utf-8") + + result = await cli_runner.invoke( + cli, ["recipes", "validate", "--path", str(recipe_path)] + ) + assert result.exit_code == 0, result.output diff --git a/tests/live/test_workspace_live.py b/tests/live/test_workspace_live.py new file mode 100644 index 0000000..d0e1d0c --- /dev/null +++ b/tests/live/test_workspace_live.py @@ -0,0 +1,24 @@ +from pathlib import Path + +import pytest + +from asyncclick.testing import CliRunner + +from tests.live.helpers import prepare_live_env, require_live_env +from workato_platform_cli.cli import cli + + +@pytest.mark.live +@pytest.mark.asyncio +async def test_workspace_live( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + require_live_env() + prepare_live_env(monkeypatch, tmp_path) + + result = await cli_runner.invoke(cli, ["workspace"]) + + assert result.exit_code == 0, result.output + assert "Current User" in result.output