diff --git a/docs/architecture.md b/docs/architecture.md index c9857444c..0485f0621 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,10 +51,9 @@ All adapters are built using user `precice` with `gid` and `uid` equal to 1000 ( This is done to ensure consistent user rights for writing and reading from mounted directories without a need to use root user on the host system. Running root within container will work, but will lead to output directory being owned by root on host. Running with different users in different directories will cause problems with access to e.g. `/home/precice/Data/Exchange` directory. More information and motivation behind the solution can be found [this blogs post](https://medium.com/@nielssj/docker-volumes-and-file-system-permissions-772c1aee23ca) and [this issue](https://github.com/moby/moby/issues/2259). -## Running tests +## The tests -At this stage, we spawn containers with the necessary solvers for the simulation. The execution is controlled using [docker-compose](https://docs.docker.com/compose/). Separate volumes are created for the input and the output of every adapter. An additional docker network is created and used for the communication between containers. Exchange of data is done using an additional, common volume. The general pattern across the tests for such volumes are paths -such as `/home/precice/Data/Exchange`, `/home/precice/Data/Input`, `/home/precice/Data/Output`, as explained above. +The actual tests are provided in `systemtests/tests`. For testing, we spawn containers with the necessary solvers for the simulation. The execution is controlled using [docker-compose](https://docs.docker.com/compose/). Separate volumes are created for the input and the output of every adapter. An additional docker network is created and used for the communication between containers. Exchange of data is done using an additional, common volume. The general pattern across the tests for such volumes are paths such as `/home/precice/Data/Exchange`, `/home/precice/Data/Input`, `/home/precice/Data/Output`, as explained above. The tests are based on the corresponding tutorials from the [tutorials repository](https://github.com/precice/tutorials). Since tests run using independent containers, several modifications are necessary to split the input to solvers in different directories, to adjust the simulated time, or to potentially change a solver's version, as well as to modify the `precice-config.xml` for the communication pattern we use in docker. This is done using the `Dockerfile.tutorial_data` image, which clones the tutorial data and adjusts and splits the input to different solvers. It is then used as the first container spawned by `docker-compose`. @@ -64,3 +63,18 @@ Since it is possible for the output of two identical tests to differ in floating the files are also compared numerically using `compare_results.sh`. This script strips away non-numerical data and then compares files field by field with respect to a specified relative tolerance (default maximum allowed difference is 1 percent) In either case after a test finishes, all the related volumes, networks and containers are removed. + +### Required environment variables + +The following environment variables are required for running the tests: + +* `SYSTEST_REMOTE`: Specifies the user on Dockerhub from which the baseimage is pulled. Example: `precice/` +* `PRECICE_BASE`: Specifies the features of the baseimage used for running the test. Example: `ubuntu1604.home-develop` or `ubuntu1804.home-develop` +* `_TAG`: Specifies the tag that will be used for the baseimage for each of the participants. Example: `latest` + +### Running the tests + +There are two possibilities for running the tests: + +1. Use `system_testing.py`: Go to `systemtests` and run `python3 system_testing.py -s ` (e.g. `python3 system_testing.py -s fe-fe --base Ubuntu1804.home`). For more information on the script run `python3 system_testing.py --help`. +2. Use `docker compose up`: Go to the corresponding test (e.g. `systemtests/tests/TestCompose_fe-fe.Ubuntu1804.home`) and run `docker-compose up`. Note that `.env` defines the previously mentioned environment variables required for running the tests. diff --git a/generate_test.py b/generate_test.py index 4d9044fd6..81771a24a 100644 --- a/generate_test.py +++ b/generate_test.py @@ -15,6 +15,7 @@ def generate_test_structure(test_name = None, base_image = None, solvers = [ a.replace('-adapter','') for a in adapters ] participants = [ p.replace(':','_') for p in participants] input_volumes = ['input_' + p for p in participants] + precice_base = '-' + base_image.lower() + '-develop' # to pass zip to jinja zipped_input = zip(participants, solvers, adapters) diff --git a/system_testing.py b/system_testing.py index 787f6720e..d91a51110 100644 --- a/system_testing.py +++ b/system_testing.py @@ -17,16 +17,16 @@ from subprocess import CalledProcessError from common import call, ccall, get_test_variants, filter_tests, get_test_participants -def build(systest, tag, branch, local, force_rebuild): +def build(systest, tag, branch, docker_username, local, force_rebuild): """ Builds a docker image for systest. """ - baseimage_name = "precice-{tag}-{branch}:latest".format(tag = tag, branch=branch) + baseimage_name = "precice-{tag}-{branch}:latest".format(tag=tag, branch=branch) test_tag = "-".join([systest, tag, branch]) docker.build_image(tag = test_tag, build_args = {"from" : docker.get_namespace() + baseimage_name if local - else 'precice/' + baseimage_name}, + else docker_username + '/' + baseimage_name}, force_rebuild = force_rebuild) def run(systest, tag, branch): @@ -37,14 +37,14 @@ def run(systest, tag, branch): shutil.rmtree("Logs", ignore_errors=True) ccall("docker cp " + test_tag + ":Output . ") -def build_adapters(systest, tag, branch, local, force_rebuild): +def build_adapters(systest, tag, branch, local, docker_username, force_rebuild): """ Builds a docker images for a preCICE adapter, participating in tests """ baseimage_name = "precice-{tag}-{branch}:latest".format(tag = tag, branch=branch) participants = get_test_participants(systest) docker_args = { 'tag': '', 'build_args': {"from": docker.get_namespace() + baseimage_name if local - else 'precice/' + baseimage_name }, + else docker_username + '/' + baseimage_name }, 'force_rebuild': force_rebuild, 'dockerfile': 'Dockerfile'} @@ -58,7 +58,7 @@ def build_adapters(systest, tag, branch, local, force_rebuild): if os.path.exists("Dockerfile.{}".format(participant)): docker.build_image(**docker_args) -def run_compose(systest, branch, local, tag, force_rebuild, rm_all=False, verbose=False): +def run_compose(systest, branch, docker_username, local, tag, force_rebuild, rm_all=False, verbose=False): """ Runs necessary systemtest with docker compose """ test_dirname = "TestCompose_{systest}".format(systest = systest) @@ -69,7 +69,7 @@ def run_compose(systest, branch, local, tag, force_rebuild, rm_all=False, verbos # set up environment variables, to detect precice base image, that we # should run with and docker images location export_cmd = "export PRECICE_BASE=-{}; ".format(adapter_base_name) - extra_cmd = "export SYSTEST_REMOTE={}; ".format(docker.get_namespace()) if local else "" + extra_cmd = "export SYSTEST_REMOTE={}; ".format(docker.get_namespace() if local else docker_username + "/") compose_config_cmd = "docker-compose config && " compose_exec_cmd = "bash ../../silent_compose.sh {}".format('debug' if verbose else "") copy_cmd = "docker cp tutorial-data:/Output ." @@ -159,22 +159,22 @@ def comparison(pathToRef, pathToOutput): sys.exit(0) -def build_run_compare(test, tag, branch, local_precice, force_rebuild, rm_all=False, verbose=False): +def build_run_compare(test, tag, branch, docker_username, local_precice, force_rebuild, rm_all=False, verbose=False): """ Runs and compares test, using precice branch. """ compose_tests = ["dealii-of", "of-of", "su2-ccx", "of-ccx", "of-of_np", "fe-fe","nutils-of", "of-ccx_fsi"] test_basename = test.split(".")[0] if local_precice: - build_adapters(test_basename, tag, branch, local_precice, force_rebuild) + build_adapters(test_basename, tag, branch, docker_username, local_precice, force_rebuild) if test_basename in compose_tests: - run_compose(test, branch, local_precice, tag, force_rebuild, rm_all, verbose) + run_compose(test, branch, docker_username, local_precice, tag, force_rebuild, rm_all, verbose) else: # remaining, non-compose tests test_dirname = "Test_{systest}".format(systest=test) test_path = os.path.join(os.getcwd(), 'tests', test_dirname) with common.chdir(test_path): # Build - build(test_basename, tag, branch, local_precice, force_rebuild) + build(test_basename, tag, branch, docker_username, local_precice, force_rebuild) run(test_basename, tag, branch) # Preparing string for path pathToRef = os.path.join(os.getcwd(), "referenceOutput") @@ -225,10 +225,11 @@ def compose_tag(docker_username, base, features, branch): parser.add_argument('-l', '--local', action='store_true', help="Use local preCICE image (default: use remote image)") parser.add_argument('-s', '--systemtest', type=str, help="Choose system tests you want to use", choices = common.get_tests(), required = True) - parser.add_argument('-b', '--branch', help="preCICE branch to use", default=os.environ["TRAVIS_BRANCH"] if os.environ["TRAVIS_PULL_REQUEST"] == "false" else os.environ["TRAVIS_PULL_REQUEST_BRANCH"]) # make sure that branch corresponding to system tests branch is used, if no branch is explicitly specified. If we are testing a pull request, make sure to test agains branch from which PR originated. + parser.add_argument('-b', '--branch', help="preCICE branch to use", default=os.environ["TRAVIS_BRANCH"] if ("TRAVIS_BRANCH" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] == "false") else (os.environ["TRAVIS_PULL_REQUEST_BRANCH"] if ("TRAVIS_PULL_REQUEST_BRANCH" in os.environ) else "develop")) # make sure that branch corresponding to system tests branch is used, if no branch is explicitly specified. If we are testing a pull request, make sure to test agains branch from which PR originated. + parser.add_argument('-u', '--docker-username', default=os.environ["DOCKER_USERNAME"] if "DOCKER_USERNAME" in os.environ else "precice", help="Name of the docker user on dockerhub that provides the preCICE and adapter base images for the tests. Defines the environment variable SYSTEST_REMOTE.") parser.add_argument('-f', '--force_rebuild', nargs='+', help="Force rebuild of variable parts of docker image", default = [], choices = ["precice", "tests"]) - parser.add_argument('--base', type=str,help="Base preCICE image to use", + parser.add_argument('--base', type=str, help="Base preCICE image to use. Defines the environmental variable PRECICE_BASE.", default= "Ubuntu1604.home") parser.add_argument('-v', '--verbose', action='store_true', help="Verbose output of participant containers") args = parser.parse_args() @@ -241,5 +242,5 @@ def compose_tag(docker_username, base, features, branch): else: test = test[0] tag = args.base.lower() - build_run_compare(test, tag, args.branch.lower(), args.local, + build_run_compare(test, tag, args.branch.lower(), args.docker_username, args.local, args.force_rebuild, rm_all=False, verbose=args.verbose) diff --git a/templates/.env.jinja b/templates/test_template/.env.jinja similarity index 76% rename from templates/.env.jinja rename to templates/test_template/.env.jinja index b360d7e01..95190ac80 100644 --- a/templates/.env.jinja +++ b/templates/test_template/.env.jinja @@ -3,4 +3,4 @@ {%- endfor %} SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE={{precice_base}} diff --git a/tests/TestCompose_fe-fe.Ubuntu1804.home/.env b/tests/TestCompose_fe-fe.Ubuntu1804.home/.env index 988195074..f2e35a1a2 100644 --- a/tests/TestCompose_fe-fe.Ubuntu1804.home/.env +++ b/tests/TestCompose_fe-fe.Ubuntu1804.home/.env @@ -1,3 +1,3 @@ FENICS_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1804.home-develop diff --git a/tests/TestCompose_nutils-of.Ubuntu1804.home/.env b/tests/TestCompose_nutils-of.Ubuntu1804.home/.env index eff569fc5..d5ba40f7a 100644 --- a/tests/TestCompose_nutils-of.Ubuntu1804.home/.env +++ b/tests/TestCompose_nutils-of.Ubuntu1804.home/.env @@ -1,3 +1,3 @@ OPENFOAM_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1804.home-develop diff --git a/tests/TestCompose_of-ccx.Ubuntu1604.home/.env b/tests/TestCompose_of-ccx.Ubuntu1604.home/.env index 22dc4f245..a3d795792 100644 --- a/tests/TestCompose_of-ccx.Ubuntu1604.home/.env +++ b/tests/TestCompose_of-ccx.Ubuntu1604.home/.env @@ -2,4 +2,4 @@ OPENFOAM_TAG=latest CALCULIX_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1604.home-develop diff --git a/tests/TestCompose_of-ccx_fsi.Ubuntu1604.home.PETSc/.env b/tests/TestCompose_of-ccx_fsi.Ubuntu1604.home.PETSc/.env index 594d0464c..50c804675 100644 --- a/tests/TestCompose_of-ccx_fsi.Ubuntu1604.home.PETSc/.env +++ b/tests/TestCompose_of-ccx_fsi.Ubuntu1604.home.PETSc/.env @@ -1,4 +1,5 @@ OPENFOAM_TAG=latest CALCULIX_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= + +PRECICE_BASE=-ubuntu1604.home.petsc-develop diff --git a/tests/TestCompose_of-of.Ubuntu1604.home/.env b/tests/TestCompose_of-of.Ubuntu1604.home/.env index eff569fc5..dead7f361 100644 --- a/tests/TestCompose_of-of.Ubuntu1604.home/.env +++ b/tests/TestCompose_of-of.Ubuntu1604.home/.env @@ -1,3 +1,3 @@ OPENFOAM_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1604.home-develop diff --git a/tests/TestCompose_of-of_np.Ubuntu1604.home/.env b/tests/TestCompose_of-of_np.Ubuntu1604.home/.env index eff569fc5..dead7f361 100644 --- a/tests/TestCompose_of-of_np.Ubuntu1604.home/.env +++ b/tests/TestCompose_of-of_np.Ubuntu1604.home/.env @@ -1,3 +1,3 @@ OPENFOAM_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1604.home-develop diff --git a/tests/TestCompose_su2-ccx.Ubuntu1604.home/.env b/tests/TestCompose_su2-ccx.Ubuntu1604.home/.env index 1953e4e18..ab314ac07 100644 --- a/tests/TestCompose_su2-ccx.Ubuntu1604.home/.env +++ b/tests/TestCompose_su2-ccx.Ubuntu1604.home/.env @@ -1,4 +1,4 @@ SU2_TAG=latest CALCULIX_TAG=latest SYSTEST_REMOTE=precice/ -PRECICE_BASE= +PRECICE_BASE=-ubuntu1604.home-develop