diff --git a/adev_config.yaml b/adev_config.yaml index 6a0bad15..93614041 100644 --- a/adev_config.yaml +++ b/adev_config.yaml @@ -22,6 +22,12 @@ autonomy_dependencies: - open-aea-ledger-cosmos - open-aea-cli-ipfs extras: null +- name: visualisation-station + version: 0.1.4 + location: remote + url: https://api.github.com/repos/StationsStation/visualisation_station + plugins: null + extras: null poetry_dependencies: - name: open-autonomy version: 0.19.4 diff --git a/auto_dev/commands/test.py b/auto_dev/commands/test.py index b242fd27..81cac598 100644 --- a/auto_dev/commands/test.py +++ b/auto_dev/commands/test.py @@ -1,7 +1,6 @@ """Command to run tests for packages.""" import rich_click as click -from rich.progress import track from auto_dev.base import build_cli from auto_dev.test import COVERAGE_COMMAND, test_path @@ -116,7 +115,8 @@ def test(ctx, path, watch, coverage_report) -> None: msg = f"Unable to get packages are you in the right directory? {error}" raise click.ClickException(msg) from error results = {} - for package in track(range(len(packages)), description="Testing..."): + for package in range(len(packages)): + click.echo(f"Testing {packages[package]} {package + 1}/{len(packages)}") result = test_path(str(packages[package]), verbose=verbose, watch=watch) results[packages[package]] = result click.echo(f"{'👌' if result else '❗'} - {packages[package]}") diff --git a/auto_dev/data/repo/templates/autonomy/.github/workflows/common_check.yaml b/auto_dev/data/repo/templates/autonomy/.github/workflows/common_check.yaml index 0ef84aef..8f3f7d17 100644 --- a/auto_dev/data/repo/templates/autonomy/.github/workflows/common_check.yaml +++ b/auto_dev/data/repo/templates/autonomy/.github/workflows/common_check.yaml @@ -20,7 +20,7 @@ jobs: matrix: python-version: # - "3.9" - - "3.10" + - "3.11" poetry-version: ["1.8.3"] os: [ubuntu-20.04,] runs-on: ${{ matrix.os }} diff --git a/auto_dev/data/repo/templates/autonomy/.github/workflows/release.yml b/auto_dev/data/repo/templates/autonomy/.github/workflows/release.yml index d5bdc21e..27d0247a 100644 --- a/auto_dev/data/repo/templates/autonomy/.github/workflows/release.yml +++ b/auto_dev/data/repo/templates/autonomy/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: - python-versions: ["3.10"] + python-versions: ["3.11"] permissions: contents: write diff --git a/auto_dev/data/repo/templates/autonomy/scripts/pre_commit_hook.sh b/auto_dev/data/repo/templates/autonomy/scripts/pre_commit_hook.sh index 31dbca02..3f410530 100644 --- a/auto_dev/data/repo/templates/autonomy/scripts/pre_commit_hook.sh +++ b/auto_dev/data/repo/templates/autonomy/scripts/pre_commit_hook.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /usr/bin/env bash # This script is used to run the pre-commit checks for the repository. # It is called by the pre-commit hook in the repository. diff --git a/auto_dev/data/repo/templates/autonomy/scripts/pre_push_hook.sh b/auto_dev/data/repo/templates/autonomy/scripts/pre_push_hook.sh index a479e2ff..dc436e6c 100644 --- a/auto_dev/data/repo/templates/autonomy/scripts/pre_push_hook.sh +++ b/auto_dev/data/repo/templates/autonomy/scripts/pre_push_hook.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /usr/bin/env bash set -e # Run lock command and store potential changes diff --git a/auto_dev/data/repo/templates/autonomy/scripts/run_mas.sh b/auto_dev/data/repo/templates/autonomy/scripts/run_mas.sh deleted file mode 100644 index 74337776..00000000 --- a/auto_dev/data/repo/templates/autonomy/scripts/run_mas.sh +++ /dev/null @@ -1,47 +0,0 @@ -#! /bin/bash -# Start the MAS service. -set -e - -echo "Using autonomy version: $(autonomy --version)" - -# we check if the service alias is already used -# if it is, we hard exit -if [ -d "service" ]; then - echo "Service $1 already exists at path ./service" - exit 1 -fi -echo "-----------------------------" -echo "Starting service $1" - -# if the key path is not set, we hard exit -if [ -z "$MAS_KEYPATH" ]; then - echo "MAS_KEYPATH is not set!" - exit 1 -fi - -echo "-----------------------------" -echo "Using keys: $MAS_KEYPATH" - -export MAS_ADDRESS=$(echo -n $(cat $MAS_KEYPATH | jq '.[].address' -r)) - -echo "Using Address: $MAS_ADDRESS" - - -# if the service name is not set, we hard exit -if [ -z "$1" ]; then - echo "Service name is not set!" - exit 1 -fi - -autonomy fetch --service $1 --local --alias service -cd service -autonomy build-image --extra-dependency hypothesis -autonomy deploy build $MAS_KEYPATH - -echo "-----------------------------" -echo "Service $1 built!" - -echo "Starting service $1" -cd abci_build && \ - docker-compose up -d --force-recreate -echo "Service $1 started!" diff --git a/auto_dev/data/templates/compose/tendermint.yaml b/auto_dev/data/templates/compose/tendermint.yaml index 62f8146d..e771d12c 100644 --- a/auto_dev/data/templates/compose/tendermint.yaml +++ b/auto_dev/data/templates/compose/tendermint.yaml @@ -6,7 +6,7 @@ services: cpus: 0.5 container_name: tm_0 hostname: tm_0 - image: "valory/open-autonomy-tendermint:0.15.2" + image: "valory/open-autonomy-tendermint:0.19.4" restart: always network_mode: ${NETWORK_MODE} environment: diff --git a/auto_dev/packages/packages.json b/auto_dev/packages/packages.json index 0612707d..75d0ca2a 100644 --- a/auto_dev/packages/packages.json +++ b/auto_dev/packages/packages.json @@ -2,6 +2,6 @@ "dev": {}, "third_party": { "agent/eightballer/base/0.1.0": "bafybeibgotyzg5n7ldgqrunowietfyqvfbmtyhm7ezpjelqiiuamnhml44", - "agent/eightballer/frontend_agent/0.1.0": "bafybeigvt6z3pcub3klzetwpmjxgzpf2hg6uq3vcoe6g7n4cdqz3jr3fqu" + "agent/eightballer/frontend_agent/0.1.0": "bafybeieq3t5vfj57hlr4tmezubut4remtpjawwtdk7aepbwpb4gun5px6e" } } \ No newline at end of file diff --git a/auto_dev/services/package_manager/index.py b/auto_dev/services/package_manager/index.py index f22f6ade..230a4856 100644 --- a/auto_dev/services/package_manager/index.py +++ b/auto_dev/services/package_manager/index.py @@ -272,7 +272,8 @@ def add_to_packages( logger.warning(f"Package already exists in dev packages: {key} skipping.") continue try: - data["dev"][key] = package_id.package_hash + key_hash = DEFAULT_IPFS_HASH if not package_id.public_id._package_hash else package_id.package_hash # noqa + data["dev"][key] = key_hash except ValueError as e: logger.exception(f"Error adding package: {e}") msg = f"Error adding package {key} to registry. {package_id}" @@ -359,5 +360,8 @@ def publish_agent( if not Path(DEFAULT_AEA_CONFIG_FILE).exists(): msg = "Not in an agent directory. Please run this command from an agent directory." raise OperationError(msg) + # we check if there are keys and remove them + if self.agent_runner: + self.agent_runner.remove_keys() # Publish from agent directory (we're already there) self._publish_internal(force, new_public_id=new_public_id) diff --git a/auto_dev/services/runner/runner.py b/auto_dev/services/runner/runner.py index 1eaaf437..7dcbfbdc 100644 --- a/auto_dev/services/runner/runner.py +++ b/auto_dev/services/runner/runner.py @@ -16,7 +16,9 @@ import requests from docker.errors import NotFound from aea.skills.base import PublicId -from aea.configurations.base import PackageType +from aea_ledger_ethereum import EthereumCrypto +from aea.helpers.env_vars import is_env_variable +from aea.configurations.base import PackageId, PackageType from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE from auto_dev.utils import change_dir, map_os_to_env_vars, load_autonolas_yaml @@ -45,6 +47,7 @@ class DevAgentRunner(AgentRunner): ipfs_hash: str | None = None use_tendermint: bool = True install_deps: bool = True + ethereum_address: str | None = None def __post_init__( self, @@ -134,7 +137,7 @@ def check_tendermint(self, retries: int = 0) -> None: time.sleep(0.2) self.check_tendermint(retries + 1) if res.status == "running": - self.attempt_hard_reset() + self._attempt_hard_reset() except (subprocess.CalledProcessError, RuntimeError, NotFound) as e: self.logger.info(f"Tendermint container not found or error: {e}") if retries > 3: @@ -151,7 +154,7 @@ def check_tendermint(self, retries: int = 0) -> None: self.logger.info("Tendermint is running and healthy ✅") return None - def attempt_hard_reset(self, attempts: int = 0) -> None: + def _attempt_hard_reset(self, attempts: int = 0) -> None: """Attempt to hard reset Tendermint.""" if attempts >= TENDERMINT_RESET_RETRIES: self.logger.error(f"Failed to reset Tendermint after {TENDERMINT_RESET_RETRIES} attempts.") @@ -168,7 +171,7 @@ def attempt_hard_reset(self, attempts: int = 0) -> None: self.logger.info(f"Tendermint not ready (attempt {attempts + 1}/{TENDERMINT_RESET_RETRIES}), waiting...") time.sleep(1) - self.attempt_hard_reset(attempts + 1) + self._attempt_hard_reset(attempts + 1) def fetch_agent(self) -> None: """Fetch the agent from registry if needed.""" @@ -202,6 +205,9 @@ def setup(self) -> None: self.logger.info("Setting up agent keys...") self.manage_keys() + self.logger.info("Detecting Necessary Overrides. ") + self.extract_magic_overrides() + self.logger.info("Installing dependencies...") self.install_dependencies() if self.install_deps else None @@ -209,6 +215,41 @@ def setup(self) -> None: self.issue_certificates() self.logger.info("Agent setup complete. 🎉") + def extract_magic_overrides(self): + """Extract the magic overrides necessary for concensus.""" + + magic_overrides = { + ".models.params.args.setup.all_participants": lambda: f'["{self.ethereum_address}"]', + } + + _config, *overrides = load_autonolas_yaml(PackageType.AGENT, self.agent_dir) + + dotted_paths = {} + # we go through each override and check if it is an imputed value + + def recurse_dictionary(dictionary, path=""): + for key, value in dictionary.items(): + if isinstance(value, dict): + recurse_dictionary(value, path=f"{path}.{key}") + elif is_env_variable(value): + dotted_paths[path + f".{key}"] = value + + for override in overrides: + _type = override["type"] + public_id = PublicId.from_str(override["public_id"]) + package_id = PackageId(package_type=PackageType(_type), public_id=public_id) + path = f"{package_id.package_type.value}.{public_id.name}" + recurse_dictionary(override, path=path) + + overrides = {} + for key in dotted_paths: + for magic_var, getter in magic_overrides.items(): + if magic_var in key: + _key = key.upper().replace(".", "_") + overrides[_key] = getter() + + self._env_vars = overrides + def manage_keys( self, generate_keys: bool = True, @@ -222,12 +263,29 @@ def manage_keys( for ledger in required_ledgers: self.logger.info(f"Processing ledger: {ledger}") # We check if a key already exists for the ledger - key_file = Path("..") / f"{ledger}_private_key.txt" + new_key_file = Path(f"{ledger}_private_key.txt") + if new_key_file.exists(): + self.logger.info(f"Key file {new_key_file} already exists. Skipping key generation.") + continue + key_file = Path("..") / new_key_file if key_file.exists(): self.setup_ledger_key(ledger, generate_keys=False, existing_key_file=key_file) else: self.setup_ledger_key(ledger, generate_keys) + self.ethereum_address = EthereumCrypto().load_private_key_from_path("ethereum_private_key.txt").address + + def remove_keys(self) -> None: + """Remove keys from the agent.""" + self.logger.info("Removing keys...") + config = load_autonolas_yaml(PackageType.AGENT)[0] + required_ledgers = config["private_key_paths"] + for ledger in required_ledgers: + cmd = f"aea -s remove-key {ledger}" + self.execute_command(cmd) + + self.logger.info("Keys removed. 🔑") + def setup_ledger_key(self, ledger: str, generate_keys, existing_key_file: Path | None = None) -> None: """Setup the agent with the ledger key.""" key_file = Path(f"{ledger}_private_key.txt") @@ -255,7 +313,7 @@ def install_dependencies(self) -> None: def issue_certificates(self) -> None: """Issue certificates for agent if needed.""" if not Path("../certs").exists(): - self.execute_command("aea -s issue-certificates") + self.execute_command("aea -s issue-certificates", verbose=True) else: self.execute_command("cp -r ../certs ./") @@ -300,7 +358,7 @@ def execute_agent( """ self.logger.info("Starting agent execution...") try: - result = self.execute_command("aea -s run --env ../.env", verbose=True) + result = self.execute_command("aea -s run --env ../.env", verbose=True, env_vars=self._env_vars) if result: self.logger.info("Agent execution completed successfully. 😎") else: diff --git a/auto_dev/test.py b/auto_dev/test.py index 1e12b726..c11b9ed4 100644 --- a/auto_dev/test.py +++ b/auto_dev/test.py @@ -4,7 +4,7 @@ from pathlib import Path from multiprocessing import cpu_count -from auto_dev.workflow_manager import Task +import pytest COVERAGE_COMMAND = f"""coverage report \ @@ -59,9 +59,7 @@ def test_path( if multiple: extra_args.extend(("-n", str(cpu_count()))) - args = ["pytest", path, *extra_args] - env = os.environ.copy() - env.update({"PYTHONPATH": "."}) - task = Task(command=" ".join(args), env_vars=env, stream=verbose, verbose=verbose) - task.work() - return not task.is_failed + args = [path, *extra_args] + os.environ["PYTHONPATH"] = "." + result = pytest.main(args) + return result == 0 diff --git a/poetry.lock b/poetry.lock index 2814b35f..803c4459 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1614,20 +1614,6 @@ files = [ [package.extras] test = ["pytest (>=6)"] -[[package]] -name = "execnet" -version = "2.1.1" -description = "execnet: rapid multi-Python deployment" -optional = false -python-versions = ">=3.8" -files = [ - {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, - {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, -] - -[package.extras] -testing = ["hatch", "pre-commit", "pytest", "tox"] - [[package]] name = "filelock" version = "3.17.0" @@ -3844,26 +3830,6 @@ files = [ packaging = ">=17.1" pytest = ">=5.3" -[[package]] -name = "pytest-xdist" -version = "3.6.1" -description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, - {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, -] - -[package.dependencies] -execnet = ">=2.1" -pytest = ">=7.0.0" - -[package.extras] -psutil = ["psutil (>=3.0)"] -setproctitle = ["setproctitle"] -testing = ["filelock"] - [[package]] name = "pytest-xprocess" version = "0.18.1" @@ -5175,4 +5141,4 @@ all = ["isort", "open-aea", "open-aea-ledger-cosmos", "open-aea-ledger-ethereum" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "d8164d6f9e3a0579dacf00b970b06c104a0bfb7119d93d9e2f8c27c29d976656" +content-hash = "cad10901bcf73748ba6f13a826812aac955564504b41359d425173c67862e6e4" diff --git a/pyproject.toml b/pyproject.toml index 3c086a66..711851ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ pycryptodome = "==3.20.0" rich-click = "^1.8.2" tbump = "^6.11.0" requests = ">=2.28.1,<3" -pytest-xdist = "^3.6.1" ruff = "^0.5.7" pydoclint = {extras = ["all"], version = "^0.6.0"} pydantic = "^2.8.2"