From da5b21c87802b34d47c678362c7f7574a6ba59d2 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 15:43:44 +0200 Subject: [PATCH 01/11] feat: [OCISDEV-774] ci, part 1 --- .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..2f6a7d3ca4f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + branches: [master, edge] + pull_request: + +jobs: + check-go-generate: + runs-on: ubuntu-latest + container: owncloudci/golang:1.24 + steps: + - uses: actions/checkout@v4 + - name: Check go generate + run: | + make go-generate + git diff --exit-code + + unit-tests: + runs-on: ubuntu-latest + container: owncloudci/golang:1.24 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: apk update && apk add --no-cache inotify-tools + - name: Run unit tests + run: make test + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage.out + + build: + needs: [unit-tests] + runs-on: ubuntu-latest + container: owncloudci/golang:1.24 + steps: + - uses: actions/checkout@v4 + - name: Build distribution + run: make dist From b71e3df49d251acaef40481d245c28b03f6c0d6a Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 15:59:23 +0200 Subject: [PATCH 02/11] feat: [OCISDEV-774] ci, part 2 --- .github/workflows/ci.yml | 153 +++++++++++- tests/acceptance/run-acceptance.py | 361 +++++++++++++++++++++++++++++ tests/acceptance/run-cs3api.py | 218 +++++++++++++++++ tests/acceptance/run-litmus.py | 218 +++++++++++++++++ 4 files changed, 938 insertions(+), 12 deletions(-) create mode 100755 tests/acceptance/run-acceptance.py create mode 100755 tests/acceptance/run-cs3api.py create mode 100755 tests/acceptance/run-litmus.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f6a7d3ca4f..861f3441da4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,29 +5,26 @@ on: branches: [master, edge] pull_request: +env: + GO_VERSION: "1.24" + jobs: check-go-generate: runs-on: ubuntu-latest container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 - - name: Check go generate - run: | - make go-generate - git diff --exit-code + - run: make go-generate && git diff --exit-code unit-tests: runs-on: ubuntu-latest container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 - - name: Install dependencies - run: apk update && apk add --no-cache inotify-tools - - name: Run unit tests - run: make test - - name: Upload coverage + - run: apk update && apk add --no-cache inotify-tools + - run: make test + - uses: actions/upload-artifact@v4 if: always() - uses: actions/upload-artifact@v4 with: name: coverage path: coverage.out @@ -38,5 +35,137 @@ jobs: container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 - - name: Build distribution - run: make dist + - run: make dist + + integration-tests: + needs: [unit-tests] + runs-on: ubuntu-latest + container: owncloudci/golang:1.24 + services: + redis: + image: redis:6-alpine + env: + REDIS_DATABASES: 1 + steps: + - uses: actions/checkout@v4 + - run: make test-integration + env: + REDIS_ADDRESS: redis:6379 + + litmus: + needs: [unit-tests] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + endpoint: [old-webdav, new-webdav, spaces-dav] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: python3 tests/acceptance/run-litmus.py --endpoint ${{ matrix.endpoint }} + + cs3api-validator: + needs: [unit-tests] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + storage: [ocis, s3ng] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: python3 tests/acceptance/run-cs3api.py --storage ${{ matrix.storage }} + + acceptance-tests-ocis: + needs: [unit-tests] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + part: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: python3 tests/acceptance/run-acceptance.py --storage ocis --total-parts 4 --run-part ${{ matrix.part }} + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: acceptance-ocis-part-${{ matrix.part }} + path: tmp/testrunner/tests/acceptance/output/ + + acceptance-tests-s3ng: + needs: [unit-tests] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + part: [1, 2, 3, 4, 5, 6] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: python3 tests/acceptance/run-acceptance.py --storage s3ng --total-parts 6 --run-part ${{ matrix.part }} + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: acceptance-s3ng-part-${{ matrix.part }} + path: tmp/testrunner/tests/acceptance/output/ + + acceptance-tests-posixfs: + needs: [unit-tests] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + part: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: python3 tests/acceptance/run-acceptance.py --storage posixfs --total-parts 4 --run-part ${{ matrix.part }} + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: acceptance-posixfs-part-${{ matrix.part }} + path: tmp/testrunner/tests/acceptance/output/ + + ci-ok: + if: always() + needs: + - check-go-generate + - unit-tests + - build + - integration-tests + - litmus + - cs3api-validator + - acceptance-tests-ocis + - acceptance-tests-s3ng + - acceptance-tests-posixfs + runs-on: ubuntu-latest + steps: + - run: | + results=( + "${{ needs.check-go-generate.result }}" + "${{ needs.unit-tests.result }}" + "${{ needs.build.result }}" + "${{ needs.integration-tests.result }}" + "${{ needs.litmus.result }}" + "${{ needs.cs3api-validator.result }}" + "${{ needs.acceptance-tests-ocis.result }}" + "${{ needs.acceptance-tests-s3ng.result }}" + "${{ needs.acceptance-tests-posixfs.result }}" + ) + for r in "${results[@]}"; do + if [[ "$r" != "success" && "$r" != "skipped" ]]; then + echo "FAILED: $r" + exit 1 + fi + done diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py new file mode 100755 index 00000000000..4c8586ad13d --- /dev/null +++ b/tests/acceptance/run-acceptance.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +""" +Run PHP Behat acceptance tests against reva with different storage backends. + +Replaces the ocis-integration-tests, s3ng-integration-tests, and +posixfs-integration-tests Drone pipelines. + +Usage: + python3 tests/acceptance/run-acceptance.py --storage ocis --total-parts 4 --run-part 1 + python3 tests/acceptance/run-acceptance.py --storage s3ng --total-parts 6 --run-part 3 + python3 tests/acceptance/run-acceptance.py --storage posixfs --total-parts 4 --run-part 2 +""" + +import argparse +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DRONE_CONFIG_DIR = REPO_ROOT / "tests" / "oc-integration-tests" / "drone" +DRONE_ENV_FILE = REPO_ROOT / ".drone.env" +REVAD_BIN = REPO_ROOT / "cmd" / "revad" / "revad" + +FRONTEND_PORT = 20080 +GATEWAY_PORT = 19000 +LDAP_PORT = 636 +REDIS_PORT = 6379 +CEPH_PORT = 4000 + +# Container images +LDAP_IMAGE = "osixia/openldap:1.3.0" +REDIS_IMAGE = "redis:6-alpine" +CEPH_IMAGE = "ceph/daemon" +PHP_IMAGE = "owncloudci/php:8.4" + +# Revad configs per storage backend +# Matches the exact configs from .drone.star for each storage type +REVAD_CONFIGS = { + "ocis": [ + "frontend.toml", + "gateway.toml", + "shares.toml", + "storage-shares.toml", + "machine-auth.toml", + "storage-users-ocis.toml", + "storage-publiclink.toml", + "permissions-ocis-ci.toml", + "ldap-users.toml", + ], + "s3ng": [ + "frontend.toml", + "gateway.toml", + "shares.toml", + "storage-users-s3ng.toml", + "storage-publiclink.toml", + "storage-shares.toml", + "ldap-users.toml", + "permissions-ocis-ci.toml", + "machine-auth.toml", + ], + "posixfs": [ + "frontend.toml", + "gateway.toml", + "shares.toml", + "storage-shares.toml", + "machine-auth.toml", + "storage-users-posixfs.toml", + "storage-publiclink.toml", + "permissions-ocis-ci.toml", + "ldap-users.toml", + ], +} + +# Storage driver names as expected by the test framework +STORAGE_DRIVER_MAP = { + "ocis": "OCIS", + "s3ng": "S3NG", + "posixfs": "POSIX", +} + +# Expected failures files per storage backend +EXPECTED_FAILURES_MAP = { + "ocis": "tests/acceptance/expected-failures-on-OCIS-storage.md", + "s3ng": "tests/acceptance/expected-failures-on-S3NG-storage.md", + "posixfs": "tests/acceptance/expected-failures-on-POSIX-storage.md", +} + +# DELETE_USER_DATA_CMD per storage backend (paths rewritten at runtime) +DELETE_CMD_MAP = { + "ocis": "rm -rf {data}/spaces/* {data}/blobs/* {data}/indexes/by-type/*", + "s3ng": "rm -rf {data}/spaces/* {data}/blobs/* {data}/indexes/by-type/*", + "posixfs": "bash -cx 'rm -rf {data}/users/* {data}/indexes/by-type/*'", +} + +# Services required per storage backend +SERVICES_NEEDED = { + "ocis": ["ldap"], + "s3ng": ["ldap", "ceph"], + "posixfs": ["redis", "ldap"], +} + + +def wait_for_port(port, timeout=60, label="service"): + """Poll until a TCP port is accepting connections.""" + start = time.time() + while time.time() - start < timeout: + try: + with socket.create_connection(("localhost", port), timeout=1): + print(f" {label} ready on port {port}") + return + except OSError: + time.sleep(0.5) + print(f"Timeout waiting for {label} on port {port} after {timeout}s", file=sys.stderr) + sys.exit(1) + + +def parse_drone_env(): + """Parse .drone.env file into a dict.""" + env = {} + if DRONE_ENV_FILE.exists(): + for line in DRONE_ENV_FILE.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + env[key.strip()] = value.strip() + return env + + +def prepare_configs(storage): + """Copy drone configs to a temp dir, rewriting hostnames and paths for GHA.""" + import tempfile + + config_dir = Path(tempfile.mkdtemp(prefix="reva-ci-config-")) + data_dir = REPO_ROOT / "tmp" / "reva" / "data" + data_dir.mkdir(parents=True, exist_ok=True) + + # Hostname/path replacements: drone container DNS -> localhost + replacements = { + "/drone/src": str(REPO_ROOT), + "ldaps://ldap:": "ldaps://localhost:", + "redis://redis:": "redis://localhost:", + "http://ceph:": "http://localhost:", + } + + for name in os.listdir(DRONE_CONFIG_DIR): + src = DRONE_CONFIG_DIR / name + dst = config_dir / name + if src.is_file(): + content = src.read_text() + for old, new in replacements.items(): + content = content.replace(old, new) + dst.write_text(content) + + return config_dir + + +def start_ldap(): + """Start LDAP service container.""" + print("Starting LDAP...") + subprocess.run( + ["docker", "run", "-d", "--name", "ldap", "--network", "host", + "-e", "LDAP_DOMAIN=owncloud.com", + "-e", "LDAP_ORGANISATION=ownCloud", + "-e", "LDAP_ADMIN_PASSWORD=admin", + "-e", "LDAP_TLS_VERIFY_CLIENT=never", + "-e", "HOSTNAME=ldap", + LDAP_IMAGE], + check=True, + ) + wait_for_port(LDAP_PORT, timeout=60, label="LDAP") + + +def start_redis(): + """Start Redis service container.""" + print("Starting Redis...") + subprocess.run( + ["docker", "run", "-d", "--name", "redis", "--network", "host", + "-e", "REDIS_DATABASES=1", + REDIS_IMAGE], + check=True, + ) + wait_for_port(REDIS_PORT, timeout=30, label="Redis") + + +def start_ceph(): + """Start Ceph demo container for S3NG storage backend.""" + print("Starting Ceph...") + subprocess.run( + ["docker", "run", "-d", "--name", "ceph", "--network", "host", + "-e", "CEPH_DAEMON=demo", + "-e", "NETWORK_AUTO_DETECT=4", + "-e", "MON_IP=0.0.0.0", + "-e", "CEPH_PUBLIC_NETWORK=0.0.0.0/0", + "-e", f"RGW_CIVETWEB_PORT={CEPH_PORT} ", + "-e", "RGW_NAME=ceph", + "-e", "CEPH_DEMO_UID=test-user", + "-e", "CEPH_DEMO_ACCESS_KEY=test", + "-e", "CEPH_DEMO_SECRET_KEY=test", + "-e", "CEPH_DEMO_BUCKET=test", + CEPH_IMAGE], + check=True, + ) + wait_for_port(CEPH_PORT, timeout=120, label="Ceph RGW") + + +SERVICE_STARTERS = { + "ldap": start_ldap, + "redis": start_redis, + "ceph": start_ceph, +} + + +def start_revad_services(config_dir, storage): + """Start all revad processes, return list of Popen objects.""" + procs = [] + for config_name in REVAD_CONFIGS[storage]: + config_path = config_dir / config_name + p = subprocess.Popen( + [str(REVAD_BIN), "-c", str(config_path)], + cwd=str(config_dir), + ) + procs.append(p) + return procs + + +def clone_test_repos(): + """Clone the ocis testing repo at the pinned commit (mirrors cloneApiTestReposStep).""" + drone_env = parse_drone_env() + commit_id = drone_env.get("APITESTS_COMMITID", "") + branch = drone_env.get("APITESTS_BRANCH", "master") + repo_url = drone_env.get("APITESTS_REPO_GIT_URL", "https://github.com/owncloud/ocis.git") + + testing_dir = REPO_ROOT / "tmp" / "testing" + testrunner_dir = REPO_ROOT / "tmp" / "testrunner" + + if not testing_dir.exists(): + print("Cloning testing repo...") + subprocess.run( + ["git", "clone", "-b", "master", "--depth=1", + "https://github.com/owncloud/testing.git", str(testing_dir)], + check=True, + ) + + if not testrunner_dir.exists(): + print(f"Cloning test runner (branch={branch}, commit={commit_id})...") + subprocess.run( + ["git", "clone", "-b", branch, "--single-branch", "--no-tags", + repo_url, str(testrunner_dir)], + check=True, + ) + if commit_id: + subprocess.run( + ["git", "checkout", commit_id], + cwd=str(testrunner_dir), + check=True, + ) + + +def run_acceptance_tests(storage, total_parts, run_part): + """Run PHP Behat acceptance tests via make test-acceptance-api.""" + data_dir = str(REPO_ROOT / "tmp" / "reva" / "data") + testrunner_dir = REPO_ROOT / "tmp" / "testrunner" + expected_failures = str(REPO_ROOT / EXPECTED_FAILURES_MAP[storage]) + + delete_cmd = DELETE_CMD_MAP[storage].format(data=data_dir) + + env = { + **os.environ, + "TEST_SERVER_URL": f"http://localhost:{FRONTEND_PORT}", + "OCIS_REVA_DATA_ROOT": f"{data_dir}/", + "DELETE_USER_DATA_CMD": delete_cmd, + "STORAGE_DRIVER": STORAGE_DRIVER_MAP[storage], + "TEST_WITH_LDAP": "true", + "REVA_LDAP_HOSTNAME": "localhost", + "TEST_REVA": "true", + "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnReva&&~@env-config", + "DIVIDE_INTO_NUM_PARTS": str(total_parts), + "RUN_PART": str(run_part), + "EXPECTED_FAILURES_FILE": expected_failures, + "ACCEPTANCE_TEST_TYPE": "core-api", + } + + print(f"Running acceptance tests: storage={storage}, part={run_part}/{total_parts}") + result = subprocess.run( + ["make", "test-acceptance-api"], + cwd=str(testrunner_dir), + env=env, + ) + return result.returncode + + +def main(): + parser = argparse.ArgumentParser(description="Run reva acceptance tests") + parser.add_argument("--storage", required=True, + choices=["ocis", "s3ng", "posixfs"], + help="Storage backend to test") + parser.add_argument("--total-parts", type=int, required=True, + help="Total number of parallel shards") + parser.add_argument("--run-part", type=int, required=True, + help="Which shard to run (1-based)") + args = parser.parse_args() + + procs = [] + docker_containers = [] + + def cleanup(*_): + for p in procs: + try: + p.terminate() + except Exception: + pass + for p in procs: + try: + p.wait(timeout=5) + except Exception: + p.kill() + for name in docker_containers: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + try: + # Build revad + print("Building revad...") + subprocess.run(["make", "build-ci"], cwd=str(REPO_ROOT), check=True) + + # Start required services + for service in SERVICES_NEEDED[args.storage]: + SERVICE_STARTERS[service]() + docker_containers.append(service) + + # Prepare configs + config_dir = prepare_configs(args.storage) + print(f"Config dir: {config_dir}") + + # Start revad services + print(f"Starting revad services ({args.storage})...") + procs = start_revad_services(config_dir, args.storage) + + # Wait for frontend and gateway to be ready + wait_for_port(FRONTEND_PORT, timeout=60, label="frontend") + wait_for_port(GATEWAY_PORT, timeout=60, label="gateway") + + # Clone test repos + clone_test_repos() + + # Run acceptance tests + rc = run_acceptance_tests(args.storage, args.total_parts, args.run_part) + sys.exit(rc) + + finally: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py new file mode 100755 index 00000000000..9f4fc2a0c4f --- /dev/null +++ b/tests/acceptance/run-cs3api.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Run CS3 API validator against reva. + +Replaces the cs3api-validator-ocis and cs3api-validator-S3NG Drone pipelines. + +Usage: + python3 tests/acceptance/run-cs3api.py --storage ocis + python3 tests/acceptance/run-cs3api.py --storage s3ng +""" + +import argparse +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DRONE_CONFIG_DIR = REPO_ROOT / "tests" / "oc-integration-tests" / "drone" +REVAD_BIN = REPO_ROOT / "cmd" / "revad" / "revad" + +CS3API_VALIDATOR_IMAGE = "owncloud/cs3api-validator:0.2.1" + +GATEWAY_PORT = 19000 +FRONTEND_PORT = 20080 + +# Revad configs per storage backend +REVAD_CONFIGS = { + "ocis": [ + "frontend.toml", + "gateway.toml", + "storage-users-ocis.toml", + "storage-shares.toml", + "storage-publiclink.toml", + "shares.toml", + "permissions-ocis-ci.toml", + "users.toml", + ], + "s3ng": [ + "frontend.toml", + "gateway.toml", + "storage-users-s3ng.toml", + "storage-shares.toml", + "storage-publiclink.toml", + "shares.toml", + "permissions-ocis-ci.toml", + "users.toml", + ], +} + +# Ceph service config (for S3NG) +CEPH_IMAGE = "ceph/daemon" +CEPH_PORT = 4000 +CEPH_ENV = { + "CEPH_DAEMON": "demo", + "NETWORK_AUTO_DETECT": "4", + "MON_IP": "0.0.0.0", + "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", + "RGW_CIVETWEB_PORT": f"{CEPH_PORT} ", + "RGW_NAME": "ceph", + "CEPH_DEMO_UID": "test-user", + "CEPH_DEMO_ACCESS_KEY": "test", + "CEPH_DEMO_SECRET_KEY": "test", + "CEPH_DEMO_BUCKET": "test", +} + + +def wait_for_port(port, timeout=60, label="service"): + """Poll until a TCP port is accepting connections.""" + start = time.time() + while time.time() - start < timeout: + try: + with socket.create_connection(("localhost", port), timeout=1): + print(f" {label} ready on port {port}") + return + except OSError: + time.sleep(0.5) + print(f"Timeout waiting for {label} on port {port} after {timeout}s", file=sys.stderr) + sys.exit(1) + + +def get_docker_bridge_ip(): + """Get the Docker bridge gateway IP for container-to-host communication.""" + r = subprocess.run( + ["docker", "network", "inspect", "bridge", + "--format", "{{range .IPAM.Config}}{{.Gateway}}{{end}}"], + capture_output=True, text=True, check=True, + ) + return r.stdout.strip() + + +def prepare_configs(storage): + """Copy drone configs to a temp dir, rewriting hostnames and paths for GHA.""" + import shutil + import tempfile + + config_dir = Path(tempfile.mkdtemp(prefix="reva-ci-config-")) + + # Hostname/path replacements: drone container DNS -> localhost + replacements = { + "/drone/src": str(REPO_ROOT), + "ldaps://ldap:": "ldaps://localhost:", + "redis://redis:": "redis://localhost:", + "http://ceph:": "http://localhost:", + } + + for name in os.listdir(DRONE_CONFIG_DIR): + src = DRONE_CONFIG_DIR / name + dst = config_dir / name + if src.is_file(): + content = src.read_text() + for old, new in replacements.items(): + content = content.replace(old, new) + dst.write_text(content) + + return config_dir + + +def start_ceph(): + """Start Ceph demo container for S3NG storage backend.""" + print("Starting Ceph...") + env_args = [] + for k, v in CEPH_ENV.items(): + env_args.extend(["-e", f"{k}={v}"]) + + subprocess.run( + ["docker", "run", "-d", "--name", "ceph", "--network", "host"] + + env_args + [CEPH_IMAGE], + check=True, + ) + # Ceph takes a while to initialize in demo mode + wait_for_port(CEPH_PORT, timeout=120, label="Ceph RGW") + + +def start_revad_services(config_dir, storage): + """Start all revad processes, return list of Popen objects.""" + procs = [] + for config_name in REVAD_CONFIGS[storage]: + config_path = config_dir / config_name + p = subprocess.Popen( + [str(REVAD_BIN), "-c", str(config_path)], + cwd=str(config_dir), + ) + procs.append(p) + return procs + + +def main(): + parser = argparse.ArgumentParser(description="Run CS3 API validator against reva") + parser.add_argument("--storage", required=True, choices=["ocis", "s3ng"], + help="Storage backend to test") + args = parser.parse_args() + + procs = [] + docker_containers = [] + + def cleanup(*_): + for p in procs: + try: + p.terminate() + except Exception: + pass + for p in procs: + try: + p.wait(timeout=5) + except Exception: + p.kill() + for name in docker_containers: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + try: + # Build revad + print("Building revad...") + subprocess.run(["make", "build-ci"], cwd=str(REPO_ROOT), check=True) + + # Start Ceph if S3NG + if args.storage == "s3ng": + start_ceph() + docker_containers.append("ceph") + + # Prepare configs + config_dir = prepare_configs(args.storage) + print(f"Config dir: {config_dir}") + + # Start revad services + print(f"Starting revad services ({args.storage})...") + procs = start_revad_services(config_dir, args.storage) + + # Wait for gateway to be ready + wait_for_port(GATEWAY_PORT, timeout=60, label="gateway") + wait_for_port(FRONTEND_PORT, timeout=60, label="frontend") + + # Get bridge IP for validator container to reach host revad + bridge_ip = get_docker_bridge_ip() + + # Run CS3 API validator + print(f"Running CS3 API validator (storage={args.storage}, endpoint={bridge_ip}:{GATEWAY_PORT})...") + result = subprocess.run( + ["docker", "run", "--rm", "--network", "host", + "--entrypoint", "/usr/bin/cs3api-validator", + CS3API_VALIDATOR_IMAGE, + "/var/lib/cs3api-validator", + f"--endpoint={bridge_ip}:{GATEWAY_PORT}"], + ) + sys.exit(result.returncode) + + finally: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tests/acceptance/run-litmus.py b/tests/acceptance/run-litmus.py new file mode 100755 index 00000000000..975cf352d63 --- /dev/null +++ b/tests/acceptance/run-litmus.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +Run litmus WebDAV compliance tests against reva. + +Replaces the litmus-ocis-old-webdav, litmus-ocis-new-webdav, and +litmus-owncloud-spaces-dav Drone pipelines. + +Usage: + python3 tests/acceptance/run-litmus.py --endpoint old-webdav + python3 tests/acceptance/run-litmus.py --endpoint new-webdav + python3 tests/acceptance/run-litmus.py --endpoint spaces-dav +""" + +import argparse +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DRONE_CONFIG_DIR = REPO_ROOT / "tests" / "oc-integration-tests" / "drone" +REVAD_BIN = REPO_ROOT / "cmd" / "revad" / "revad" + +LITMUS_IMAGE = "owncloud/litmus:latest" +LITMUS_USERNAME = "einstein" +LITMUS_PASSWORD = "relativity" +LITMUS_TESTS = "basic http copymove props" + +# einstein user UUID (see https://github.com/owncloud/ocis-accounts/blob/8de0530f31ed5ffb0bbb7f7f3471f87f429cb2ea/pkg/service/v0/service.go#L45) +EINSTEIN_UUID = "4c510ada-c86b-4815-8820-42cdf82c3d51" + +# Revad configs for litmus tests (same set used by all three endpoints) +REVAD_CONFIGS = [ + "frontend.toml", + "gateway.toml", + "storage-users-ocis.toml", + "storage-shares.toml", + "storage-publiclink.toml", + "shares.toml", + "permissions-ocis-ci.toml", + "users.toml", +] + +FRONTEND_PORT = 20080 +GATEWAY_PORT = 19000 + + +def wait_for_port(port, timeout=60, label="service"): + """Poll until a TCP port is accepting connections.""" + start = time.time() + while time.time() - start < timeout: + try: + with socket.create_connection(("localhost", port), timeout=1): + print(f" {label} ready on port {port}") + return + except OSError: + time.sleep(0.5) + print(f"Timeout waiting for {label} on port {port} after {timeout}s", file=sys.stderr) + sys.exit(1) + + +def get_docker_bridge_ip(): + """Get the Docker bridge gateway IP for container-to-host communication.""" + r = subprocess.run( + ["docker", "network", "inspect", "bridge", + "--format", "{{range .IPAM.Config}}{{.Gateway}}{{end}}"], + capture_output=True, text=True, check=True, + ) + return r.stdout.strip() + + +def prepare_configs(): + """Copy drone configs to a temp dir, rewriting /drone/src paths to repo root.""" + import shutil + import tempfile + + config_dir = Path(tempfile.mkdtemp(prefix="reva-ci-config-")) + for name in os.listdir(DRONE_CONFIG_DIR): + src = DRONE_CONFIG_DIR / name + dst = config_dir / name + if src.is_file(): + content = src.read_text() + content = content.replace("/drone/src", str(REPO_ROOT)) + dst.write_text(content) + return config_dir + + +def start_revad_services(config_dir): + """Start all revad processes, return list of Popen objects.""" + procs = [] + for config_name in REVAD_CONFIGS: + config_path = config_dir / config_name + p = subprocess.Popen( + [str(REVAD_BIN), "-c", str(config_path)], + cwd=str(config_dir), + ) + procs.append(p) + return procs + + +def get_space_id(bridge_ip): + """Get the personal space ID for einstein user via PROPFIND.""" + base = f"http://{bridge_ip}:{FRONTEND_PORT}" + + # Trigger home creation by accessing the files endpoint + subprocess.run( + ["curl", "-s", "-k", "-u", f"{LITMUS_USERNAME}:{LITMUS_PASSWORD}", + "-I", f"{base}/remote.php/dav/files/{LITMUS_USERNAME}"], + capture_output=True, + ) + time.sleep(1) + + # Get space ID via PROPFIND + r = subprocess.run( + ["curl", "-XPROPFIND", "-s", "-k", + "-u", f"{LITMUS_USERNAME}:{LITMUS_PASSWORD}", + f"{base}/dav/files/{LITMUS_USERNAME}"], + capture_output=True, text=True, + ) + # Parse space ID from oc:spaceid element + import re + match = re.search(r"([^<]+)", r.stdout) + if not match: + # Fallback: try awk-style extraction + for line in r.stdout.split("\n"): + if "oc:spaceid" in line: + parts = line.split("<") + for part in parts: + if part.startswith("oc:spaceid>"): + return part.split(">")[1].split("<")[0] + print(f"Could not extract space ID from PROPFIND response", file=sys.stderr) + print(f"Response: {r.stdout[:500]}", file=sys.stderr) + sys.exit(1) + return match.group(1) + + +def run_litmus(endpoint_url, bridge_ip): + """Run litmus Docker container against the given endpoint.""" + result = subprocess.run( + ["docker", "run", "--rm", "--network", "host", + "-e", f"LITMUS_URL={endpoint_url}", + "-e", f"LITMUS_USERNAME={LITMUS_USERNAME}", + "-e", f"LITMUS_PASSWORD={LITMUS_PASSWORD}", + "-e", f"TESTS={LITMUS_TESTS}", + LITMUS_IMAGE], + ) + return result.returncode + + +def main(): + parser = argparse.ArgumentParser(description="Run litmus WebDAV tests against reva") + parser.add_argument("--endpoint", required=True, + choices=["old-webdav", "new-webdav", "spaces-dav"], + help="Which litmus endpoint variant to test") + args = parser.parse_args() + + procs = [] + + def cleanup(*_): + for p in procs: + try: + p.terminate() + except Exception: + pass + # Wait briefly for processes to exit + for p in procs: + try: + p.wait(timeout=5) + except Exception: + p.kill() + + signal.signal(signal.SIGTERM, cleanup) + signal.signal(signal.SIGINT, cleanup) + + try: + # Build revad + print("Building revad...") + subprocess.run(["make", "build-ci"], cwd=str(REPO_ROOT), check=True) + + # Prepare configs (rewrite /drone/src paths) + config_dir = prepare_configs() + print(f"Config dir: {config_dir}") + + # Start revad services + print("Starting revad services...") + procs = start_revad_services(config_dir) + + # Wait for frontend and gateway to be ready + wait_for_port(FRONTEND_PORT, timeout=60, label="frontend") + wait_for_port(GATEWAY_PORT, timeout=60, label="gateway") + + # Get Docker bridge IP for container-to-host communication + bridge_ip = get_docker_bridge_ip() + base_url = f"http://{bridge_ip}:{FRONTEND_PORT}" + + # Determine litmus URL based on endpoint type + if args.endpoint == "old-webdav": + litmus_url = f"{base_url}/remote.php/webdav" + elif args.endpoint == "new-webdav": + litmus_url = f"{base_url}/remote.php/dav/files/{EINSTEIN_UUID}" + elif args.endpoint == "spaces-dav": + space_id = get_space_id(bridge_ip) + litmus_url = f"{base_url}/remote.php/dav/spaces/{space_id}" + print(f"Space ID: {space_id}") + + print(f"Running litmus: {args.endpoint} -> {litmus_url}") + rc = run_litmus(litmus_url, bridge_ip) + sys.exit(rc) + + finally: + cleanup() + + +if __name__ == "__main__": + main() From fb7f88a464ae2627f95fe8436bc8903d87761c59 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 18:26:43 +0200 Subject: [PATCH 03/11] ci: trigger CI From 59c2f2abfc0d7d52a3849e7afecf7dfc0aa42683 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 18:34:02 +0200 Subject: [PATCH 04/11] feat: [OCISDEV-774] ci --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 861f3441da4..e0b66566108 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ jobs: container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: make go-generate && git diff --exit-code unit-tests: @@ -21,6 +22,7 @@ jobs: container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: apk update && apk add --no-cache inotify-tools - run: make test - uses: actions/upload-artifact@v4 @@ -35,6 +37,7 @@ jobs: container: owncloudci/golang:1.24 steps: - uses: actions/checkout@v4 + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: make dist integration-tests: @@ -48,6 +51,7 @@ jobs: REDIS_DATABASES: 1 steps: - uses: actions/checkout@v4 + - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: make test-integration env: REDIS_ADDRESS: redis:6379 From d80b6709905ef2a75e40d7b9c702e5878376b506 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 18:59:03 +0200 Subject: [PATCH 05/11] feat: [OCISDEV-774] ci --- .github/workflows/ci.yml | 9 +++++++++ tests/acceptance/run-acceptance.py | 1 + tests/acceptance/run-cs3api.py | 8 ++------ tests/acceptance/run-litmus.py | 13 ++++++++----- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0b66566108..8a4fb790d5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,9 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.4" - run: python3 tests/acceptance/run-acceptance.py --storage ocis --total-parts 4 --run-part ${{ matrix.part }} - uses: actions/upload-artifact@v4 if: failure() @@ -115,6 +118,9 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.4" - run: python3 tests/acceptance/run-acceptance.py --storage s3ng --total-parts 6 --run-part ${{ matrix.part }} - uses: actions/upload-artifact@v4 if: failure() @@ -134,6 +140,9 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.4" - run: python3 tests/acceptance/run-acceptance.py --storage posixfs --total-parts 4 --run-part ${{ matrix.part }} - uses: actions/upload-artifact@v4 if: failure() diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index 4c8586ad13d..c44716be4ec 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -141,6 +141,7 @@ def prepare_configs(storage): # Hostname/path replacements: drone container DNS -> localhost replacements = { "/drone/src": str(REPO_ROOT), + "revad-services": "localhost", "ldaps://ldap:": "ldaps://localhost:", "redis://redis:": "redis://localhost:", "http://ceph:": "http://localhost:", diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index 9f4fc2a0c4f..881dccd0655 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -93,15 +93,11 @@ def get_docker_bridge_ip(): def prepare_configs(storage): - """Copy drone configs to a temp dir, rewriting hostnames and paths for GHA.""" - import shutil - import tempfile - config_dir = Path(tempfile.mkdtemp(prefix="reva-ci-config-")) - - # Hostname/path replacements: drone container DNS -> localhost + (REPO_ROOT / "tmp" / "reva" / "data").mkdir(parents=True, exist_ok=True) replacements = { "/drone/src": str(REPO_ROOT), + "revad-services": "localhost", "ldaps://ldap:": "ldaps://localhost:", "redis://redis:": "redis://localhost:", "http://ceph:": "http://localhost:", diff --git a/tests/acceptance/run-litmus.py b/tests/acceptance/run-litmus.py index 975cf352d63..827b4c3f9cf 100755 --- a/tests/acceptance/run-litmus.py +++ b/tests/acceptance/run-litmus.py @@ -17,6 +17,7 @@ import socket import subprocess import sys +import tempfile import time from pathlib import Path @@ -73,17 +74,19 @@ def get_docker_bridge_ip(): def prepare_configs(): - """Copy drone configs to a temp dir, rewriting /drone/src paths to repo root.""" - import shutil - import tempfile - config_dir = Path(tempfile.mkdtemp(prefix="reva-ci-config-")) + (REPO_ROOT / "tmp" / "reva" / "data").mkdir(parents=True, exist_ok=True) + replacements = { + "/drone/src": str(REPO_ROOT), + "revad-services": "localhost", + } for name in os.listdir(DRONE_CONFIG_DIR): src = DRONE_CONFIG_DIR / name dst = config_dir / name if src.is_file(): content = src.read_text() - content = content.replace("/drone/src", str(REPO_ROOT)) + for old, new in replacements.items(): + content = content.replace(old, new) dst.write_text(content) return config_dir From 82437e521075fd7242509323db9fff64aac6c768 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 19:18:52 +0200 Subject: [PATCH 06/11] feat: [OCISDEV-774] ci --- .github/workflows/ci.yml | 1 + tests/acceptance/run-acceptance.py | 3 ++- tests/acceptance/run-cs3api.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4fb790d5a..612550b68fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,6 +143,7 @@ jobs: - uses: shivammathur/setup-php@v2 with: php-version: "8.4" + - run: sudo apt-get update && sudo apt-get install -y inotify-tools - run: python3 tests/acceptance/run-acceptance.py --storage posixfs --total-parts 4 --run-part ${{ matrix.part }} - uses: actions/upload-artifact@v4 if: failure() diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index c44716be4ec..9cdd7f7756c 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -29,7 +29,7 @@ GATEWAY_PORT = 19000 LDAP_PORT = 636 REDIS_PORT = 6379 -CEPH_PORT = 4000 +CEPH_PORT = 8080 # Container images LDAP_IMAGE = "osixia/openldap:1.3.0" @@ -282,6 +282,7 @@ def run_acceptance_tests(storage, total_parts, run_part): "DIVIDE_INTO_NUM_PARTS": str(total_parts), "RUN_PART": str(run_part), "EXPECTED_FAILURES_FILE": expected_failures, + "REGULAR_USER_PASSWORD": "relativity", "ACCEPTANCE_TEST_TYPE": "core-api", } diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index 881dccd0655..afdfce5fe0e 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -15,6 +15,7 @@ import socket import subprocess import sys +import tempfile import time from pathlib import Path @@ -53,7 +54,7 @@ # Ceph service config (for S3NG) CEPH_IMAGE = "ceph/daemon" -CEPH_PORT = 4000 +CEPH_PORT = 8080 CEPH_ENV = { "CEPH_DAEMON": "demo", "NETWORK_AUTO_DETECT": "4", From 928fa30cb377822b713b3c9b7785cbbbe7d6cbba Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 20:03:20 +0200 Subject: [PATCH 07/11] feat: [OCISDEV-774] ci --- tests/acceptance/run-acceptance.py | 1 - tests/acceptance/run-cs3api.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index 9cdd7f7756c..8c0a5fd6f7a 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -196,7 +196,6 @@ def start_ceph(): "-e", "NETWORK_AUTO_DETECT=4", "-e", "MON_IP=0.0.0.0", "-e", "CEPH_PUBLIC_NETWORK=0.0.0.0/0", - "-e", f"RGW_CIVETWEB_PORT={CEPH_PORT} ", "-e", "RGW_NAME=ceph", "-e", "CEPH_DEMO_UID=test-user", "-e", "CEPH_DEMO_ACCESS_KEY=test", diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index afdfce5fe0e..360e623ec4b 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -60,7 +60,6 @@ "NETWORK_AUTO_DETECT": "4", "MON_IP": "0.0.0.0", "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", - "RGW_CIVETWEB_PORT": f"{CEPH_PORT} ", "RGW_NAME": "ceph", "CEPH_DEMO_UID": "test-user", "CEPH_DEMO_ACCESS_KEY": "test", From d1b610f164e1cfed07c97c076ec9f835a0b31a7c Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 20:52:19 +0200 Subject: [PATCH 08/11] feat: [OCISDEV-774] ci --- tests/acceptance/run-acceptance.py | 26 ++++++++++++++++++++------ tests/acceptance/run-cs3api.py | 27 ++++++++++++++++++++------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index 8c0a5fd6f7a..44a7d8abbb7 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -105,17 +105,23 @@ def wait_for_port(port, timeout=60, label="service"): - """Poll until a TCP port is accepting connections.""" start = time.time() while time.time() - start < timeout: try: with socket.create_connection(("localhost", port), timeout=1): - print(f" {label} ready on port {port}") return except OSError: time.sleep(0.5) - print(f"Timeout waiting for {label} on port {port} after {timeout}s", file=sys.stderr) - sys.exit(1) + sys.exit(f"Timeout waiting for {label} on port {port} after {timeout}s") + + +def wait_for_http(check_fn, timeout=120, label="service"): + start = time.time() + while time.time() - start < timeout: + if check_fn(): + return + time.sleep(1) + sys.exit(f"Timeout waiting for {label} after {timeout}s") def parse_drone_env(): @@ -187,8 +193,16 @@ def start_redis(): wait_for_port(REDIS_PORT, timeout=30, label="Redis") +def ceph_bucket_ready(): + """Check that Ceph RGW is up AND the demo bucket exists.""" + r = subprocess.run( + ["curl", "-sf", f"http://localhost:{CEPH_PORT}/test"], + capture_output=True, + ) + return r.returncode == 0 + + def start_ceph(): - """Start Ceph demo container for S3NG storage backend.""" print("Starting Ceph...") subprocess.run( ["docker", "run", "-d", "--name", "ceph", "--network", "host", @@ -204,7 +218,7 @@ def start_ceph(): CEPH_IMAGE], check=True, ) - wait_for_port(CEPH_PORT, timeout=120, label="Ceph RGW") + wait_for_http(ceph_bucket_ready, timeout=180, label="Ceph RGW bucket") SERVICE_STARTERS = { diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index 360e623ec4b..ceabb5ed625 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -69,17 +69,23 @@ def wait_for_port(port, timeout=60, label="service"): - """Poll until a TCP port is accepting connections.""" start = time.time() while time.time() - start < timeout: try: with socket.create_connection(("localhost", port), timeout=1): - print(f" {label} ready on port {port}") return except OSError: time.sleep(0.5) - print(f"Timeout waiting for {label} on port {port} after {timeout}s", file=sys.stderr) - sys.exit(1) + sys.exit(f"Timeout waiting for {label} on port {port} after {timeout}s") + + +def wait_for_http(check_fn, timeout=120, label="service"): + start = time.time() + while time.time() - start < timeout: + if check_fn(): + return + time.sleep(1) + sys.exit(f"Timeout waiting for {label} after {timeout}s") def get_docker_bridge_ip(): @@ -115,8 +121,16 @@ def prepare_configs(storage): return config_dir +def ceph_bucket_ready(): + """Check that Ceph RGW is up AND the demo bucket exists.""" + r = subprocess.run( + ["curl", "-sf", f"http://localhost:{CEPH_PORT}/test"], + capture_output=True, + ) + return r.returncode == 0 + + def start_ceph(): - """Start Ceph demo container for S3NG storage backend.""" print("Starting Ceph...") env_args = [] for k, v in CEPH_ENV.items(): @@ -127,8 +141,7 @@ def start_ceph(): env_args + [CEPH_IMAGE], check=True, ) - # Ceph takes a while to initialize in demo mode - wait_for_port(CEPH_PORT, timeout=120, label="Ceph RGW") + wait_for_http(ceph_bucket_ready, timeout=180, label="Ceph RGW bucket") def start_revad_services(config_dir, storage): From 9ab4c7983f3c388d61263fac142dd0c72041063d Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 21:10:25 +0200 Subject: [PATCH 09/11] feat: [OCISDEV-774] ci --- tests/acceptance/run-acceptance.py | 12 ++++++------ tests/acceptance/run-cs3api.py | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index 44a7d8abbb7..fb9670c7bcb 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -193,13 +193,12 @@ def start_redis(): wait_for_port(REDIS_PORT, timeout=30, label="Redis") -def ceph_bucket_ready(): - """Check that Ceph RGW is up AND the demo bucket exists.""" +def ceph_ready(): r = subprocess.run( - ["curl", "-sf", f"http://localhost:{CEPH_PORT}/test"], - capture_output=True, + ["curl", "-s", f"http://localhost:{CEPH_PORT}", "-o", "/dev/null", "-w", "%{http_code}"], + capture_output=True, text=True, ) - return r.returncode == 0 + return r.stdout.strip() == "200" def start_ceph(): @@ -218,7 +217,8 @@ def start_ceph(): CEPH_IMAGE], check=True, ) - wait_for_http(ceph_bucket_ready, timeout=180, label="Ceph RGW bucket") + wait_for_http(ceph_ready, timeout=180, label="Ceph RGW") + time.sleep(10) # allow demo bucket creation after RGW is ready SERVICE_STARTERS = { diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index ceabb5ed625..d277cd17ef9 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -121,13 +121,12 @@ def prepare_configs(storage): return config_dir -def ceph_bucket_ready(): - """Check that Ceph RGW is up AND the demo bucket exists.""" +def ceph_ready(): r = subprocess.run( - ["curl", "-sf", f"http://localhost:{CEPH_PORT}/test"], - capture_output=True, + ["curl", "-s", f"http://localhost:{CEPH_PORT}", "-o", "/dev/null", "-w", "%{http_code}"], + capture_output=True, text=True, ) - return r.returncode == 0 + return r.stdout.strip() == "200" def start_ceph(): @@ -141,7 +140,8 @@ def start_ceph(): env_args + [CEPH_IMAGE], check=True, ) - wait_for_http(ceph_bucket_ready, timeout=180, label="Ceph RGW bucket") + wait_for_http(ceph_ready, timeout=180, label="Ceph RGW") + time.sleep(10) # allow demo bucket creation after RGW is ready def start_revad_services(config_dir, storage): From e8e3cf221b7eba81634235d291b838fc3bfc3a9a Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 21:38:31 +0200 Subject: [PATCH 10/11] feat: [OCISDEV-774] ci --- tests/acceptance/run-acceptance.py | 22 ++++------------------ tests/acceptance/run-cs3api.py | 22 ++++------------------ 2 files changed, 8 insertions(+), 36 deletions(-) diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index fb9670c7bcb..91337eb6350 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -115,14 +115,6 @@ def wait_for_port(port, timeout=60, label="service"): sys.exit(f"Timeout waiting for {label} on port {port} after {timeout}s") -def wait_for_http(check_fn, timeout=120, label="service"): - start = time.time() - while time.time() - start < timeout: - if check_fn(): - return - time.sleep(1) - sys.exit(f"Timeout waiting for {label} after {timeout}s") - def parse_drone_env(): """Parse .drone.env file into a dict.""" @@ -193,14 +185,6 @@ def start_redis(): wait_for_port(REDIS_PORT, timeout=30, label="Redis") -def ceph_ready(): - r = subprocess.run( - ["curl", "-s", f"http://localhost:{CEPH_PORT}", "-o", "/dev/null", "-w", "%{http_code}"], - capture_output=True, text=True, - ) - return r.stdout.strip() == "200" - - def start_ceph(): print("Starting Ceph...") subprocess.run( @@ -209,6 +193,7 @@ def start_ceph(): "-e", "NETWORK_AUTO_DETECT=4", "-e", "MON_IP=0.0.0.0", "-e", "CEPH_PUBLIC_NETWORK=0.0.0.0/0", + "-e", f"RGW_CIVETWEB_PORT={CEPH_PORT}", "-e", "RGW_NAME=ceph", "-e", "CEPH_DEMO_UID=test-user", "-e", "CEPH_DEMO_ACCESS_KEY=test", @@ -217,8 +202,9 @@ def start_ceph(): CEPH_IMAGE], check=True, ) - wait_for_http(ceph_ready, timeout=180, label="Ceph RGW") - time.sleep(10) # allow demo bucket creation after RGW is ready + wait_for_port(CEPH_PORT, timeout=180, label="Ceph RGW") + # Wait for demo bucket creation after RGW starts accepting connections + time.sleep(15) SERVICE_STARTERS = { diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index d277cd17ef9..7be3431b226 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -60,6 +60,7 @@ "NETWORK_AUTO_DETECT": "4", "MON_IP": "0.0.0.0", "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", + "RGW_CIVETWEB_PORT": str(CEPH_PORT), "RGW_NAME": "ceph", "CEPH_DEMO_UID": "test-user", "CEPH_DEMO_ACCESS_KEY": "test", @@ -79,14 +80,6 @@ def wait_for_port(port, timeout=60, label="service"): sys.exit(f"Timeout waiting for {label} on port {port} after {timeout}s") -def wait_for_http(check_fn, timeout=120, label="service"): - start = time.time() - while time.time() - start < timeout: - if check_fn(): - return - time.sleep(1) - sys.exit(f"Timeout waiting for {label} after {timeout}s") - def get_docker_bridge_ip(): """Get the Docker bridge gateway IP for container-to-host communication.""" @@ -121,14 +114,6 @@ def prepare_configs(storage): return config_dir -def ceph_ready(): - r = subprocess.run( - ["curl", "-s", f"http://localhost:{CEPH_PORT}", "-o", "/dev/null", "-w", "%{http_code}"], - capture_output=True, text=True, - ) - return r.stdout.strip() == "200" - - def start_ceph(): print("Starting Ceph...") env_args = [] @@ -140,8 +125,9 @@ def start_ceph(): env_args + [CEPH_IMAGE], check=True, ) - wait_for_http(ceph_ready, timeout=180, label="Ceph RGW") - time.sleep(10) # allow demo bucket creation after RGW is ready + wait_for_port(CEPH_PORT, timeout=180, label="Ceph RGW") + # Wait for demo bucket creation after RGW starts accepting connections + time.sleep(15) def start_revad_services(config_dir, storage): From ad7c76ad15ae02a57a6c91d9cf33cdfb87b12f39 Mon Sep 17 00:00:00 2001 From: Michal Klos Date: Tue, 31 Mar 2026 22:07:32 +0200 Subject: [PATCH 11/11] feat: [OCISDEV-774] ci --- tests/acceptance/run-acceptance.py | 5 +++-- tests/acceptance/run-cs3api.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/acceptance/run-acceptance.py b/tests/acceptance/run-acceptance.py index 91337eb6350..40287c8650e 100755 --- a/tests/acceptance/run-acceptance.py +++ b/tests/acceptance/run-acceptance.py @@ -188,9 +188,10 @@ def start_redis(): def start_ceph(): print("Starting Ceph...") subprocess.run( - ["docker", "run", "-d", "--name", "ceph", "--network", "host", + ["docker", "run", "-d", "--name", "ceph", + "-p", f"{CEPH_PORT}:{CEPH_PORT}", "-e", "CEPH_DAEMON=demo", - "-e", "NETWORK_AUTO_DETECT=4", + "-e", "NETWORK_AUTO_DETECT=1", "-e", "MON_IP=0.0.0.0", "-e", "CEPH_PUBLIC_NETWORK=0.0.0.0/0", "-e", f"RGW_CIVETWEB_PORT={CEPH_PORT}", diff --git a/tests/acceptance/run-cs3api.py b/tests/acceptance/run-cs3api.py index 7be3431b226..21f5a6f06ca 100755 --- a/tests/acceptance/run-cs3api.py +++ b/tests/acceptance/run-cs3api.py @@ -57,7 +57,7 @@ CEPH_PORT = 8080 CEPH_ENV = { "CEPH_DAEMON": "demo", - "NETWORK_AUTO_DETECT": "4", + "NETWORK_AUTO_DETECT": "1", "MON_IP": "0.0.0.0", "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", "RGW_CIVETWEB_PORT": str(CEPH_PORT), @@ -121,7 +121,8 @@ def start_ceph(): env_args.extend(["-e", f"{k}={v}"]) subprocess.run( - ["docker", "run", "-d", "--name", "ceph", "--network", "host"] + + ["docker", "run", "-d", "--name", "ceph", + "-p", f"{CEPH_PORT}:{CEPH_PORT}"] + env_args + [CEPH_IMAGE], check=True, )