From 3a85409844f034d00ad1314f3133155efded974c Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Tue, 12 Sep 2023 11:44:38 +0100 Subject: [PATCH 01/25] First attempt to use ols4 and repair the build with new Docker repo main branch name. --- .gitmodules | 2 +- tests/test_basic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index acbf2ab..44a1bf9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "tests/ols-docker"] path = tests/ols-docker url = https://github.com/Ensembl/OLS-docker.git - branch = master + branch = main diff --git a/tests/test_basic.py b/tests/test_basic.py index 10fd15a..3bbb389 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -484,7 +484,7 @@ def fetch_document(self, path, params=None, filters=None, base_document=None): BfoClientMixin.count_call = BfoClientMixin.count_call + 1 return super().fetch_document(path, params, filters, base_document) - terms_list_client = BfoClientMixin('/'.join(['https://www.ebi.ac.uk/ols/api', 'ontologies', 'pr']), + terms_list_client = BfoClientMixin('/'.join(['https://www.ebi.ac.uk/ols4/api', 'ontologies', 'pr']), helpers.Term, page_size=100) terms = terms_list_client() From 947489112e2885ffe9ab3cfd053bc96e37416383 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Tue, 12 Sep 2023 12:12:23 +0100 Subject: [PATCH 02/25] ols4 breaking a search. --- tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index 3bbb389..10fd15a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -484,7 +484,7 @@ def fetch_document(self, path, params=None, filters=None, base_document=None): BfoClientMixin.count_call = BfoClientMixin.count_call + 1 return super().fetch_document(path, params, filters, base_document) - terms_list_client = BfoClientMixin('/'.join(['https://www.ebi.ac.uk/ols4/api', 'ontologies', 'pr']), + terms_list_client = BfoClientMixin('/'.join(['https://www.ebi.ac.uk/ols/api', 'ontologies', 'pr']), helpers.Term, page_size=100) terms = terms_list_client() From 3c2db0d83a9a2c6ca07cc4fdca5bf7a3ec17332a Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 13 Nov 2023 11:37:02 +0000 Subject: [PATCH 03/25] Removed submodule --- .gitmodules | 4 ---- tests/ols-docker | 1 - 2 files changed, 5 deletions(-) delete mode 160000 tests/ols-docker diff --git a/.gitmodules b/.gitmodules index 44a1bf9..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "tests/ols-docker"] - path = tests/ols-docker - url = https://github.com/Ensembl/OLS-docker.git - branch = main diff --git a/tests/ols-docker b/tests/ols-docker deleted file mode 160000 index 5f33506..0000000 --- a/tests/ols-docker +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5f335069840600329ab33b18d83b0df3cf18eec2 From 870cea12c3f776be3c723a186859b26e8993ff8a Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 17:34:09 +0000 Subject: [PATCH 04/25] Test Travis with new docker container. --- .gitmodules | 0 .travis.yml | 6 +-- tests/ols-config.json | 92 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) delete mode 100644 .gitmodules create mode 100644 tests/ols-config.json diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/.travis.yml b/.travis.yml index 6329fb6..223b4f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,7 @@ python: services: - docker before_install: - - git submodule sync - - git submodule update --init --recursive --remote - - docker build -t ols tests/ols-docker - - docker run -d -p 127.0.0.1:8080:8080 -t ols - - docker ps -a + - docker run -it --privileged ensemblorg/ensembl-ols-docker:latest -p 8080:8080 install: - pip install -e . - pip install -r requirements-test.txt diff --git a/tests/ols-config.json b/tests/ols-config.json new file mode 100644 index 0000000..3c1953f --- /dev/null +++ b/tests/ols-config.json @@ -0,0 +1,92 @@ +{ + "ontologies": [ + { + "activity_status": "inactive", + "build": { + "method": "owl2obo", + "source_url": "http://purl.obolibrary.org/obo/aero.owl" + }, + "contact": { + "email": "mcourtot@gmail.com", + "label": "Melanie Courtot", + "orcid": "0000-0002-9551-6370" + }, + "description": "The Adverse Event Reporting Ontology (AERO) is an ontology aimed at supporting clinicians at the time of data entry, increasing quality and accuracy of reported adverse events", + "domain": "health", + "homepage": "http://purl.obolibrary.org/obo/aero", + "id": "aero", + "is_obsolete": true, + "layout": "ontology_detail", + "license": { + "label": "CC BY 3.0", + "logo": "http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png", + "url": "http://creativecommons.org/licenses/by/3.0/" + }, + "products": [ + { + "id": "aero.owl", + "ontology_purl": "http://purl.obolibrary.org/obo/aero.owl" + } + ], + "title": "Adverse Event Reporting Ontology" + }, + { + "activity_status": "active", + "browsers": [ + { + "label": "BioPortal", + "title": "BioPortal Browser", + "url": "http://bioportal.bioontology.org/ontologies/BFO?p=classes" + } + ], + "contact": { + "email": "phismith@buffalo.edu", + "github": "phismith", + "label": "Barry Smith", + "orcid": "0000-0003-1384-116X" + }, + "depicted_by": "https://avatars2.githubusercontent.com/u/12972134?v=3&s=200", + "description": "The upper level ontology upon which OBO Foundry ontologies are built.", + "domain": "upper", + "homepage": "http://ifomis.org/bfo/", + "id": "bfo", + "in_foundry_order": 1, + "layout": "ontology_detail", + "license": { + "label": "CC BY 4.0", + "logo": "http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "mailing_list": "https://groups.google.com/forum/#!forum/bfo-discuss", + "ontology_purl": "http://purl.obolibrary.org/obo/bfo.owl", + "preferredPrefix": "BFO", + "products": [ + { + "id": "bfo.owl", + "ontology_purl": "http://purl.obolibrary.org/obo/bfo.owl" + }, + { + "id": "bfo.obo", + "ontology_purl": "http://purl.obolibrary.org/obo/bfo.obo" + } + ], + "repository": "https://github.com/BFO-ontology/BFO", + "review": { + "date": 2016, + "document": { + "label": "PDF", + "link": "https://drive.google.com/open?id=0B81h9ah4tAM_RnNTRUZnVGRyWXM" + } + }, + "title": "Basic Formal Ontology", + "tracker": "https://github.com/BFO-ontology/BFO/issues", + "usages": [ + { + "description": "BFO is imported by multiple OBO ontologies to standardize upper level structure", + "type": "owl_import", + "user": "http://obofoundry.org" + } + ] + } + ] +} From 4c9874941475de77aa085b32a3b578b30a2a64ee Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 18:08:01 +0000 Subject: [PATCH 05/25] Silent docker compose to reduce amount of logs in travis. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 223b4f8..8c26c8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: services: - docker before_install: - - docker run -it --privileged ensemblorg/ensembl-ols-docker:latest -p 8080:8080 + - docker run --pull=always -it --privileged ensemblorg/ensembl-ols-docker:latest -p 8080:8080 install: - pip install -e . - pip install -r requirements-test.txt From 0f7ae3ba7a163529d13d24d1fe9cf32a36114333 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:03:25 +0000 Subject: [PATCH 06/25] Attempt to run directly from repo. --- .gitmodules | 4 ++++ .travis.yml | 6 +++++- tests/ols-docker | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 tests/ols-docker diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..eecda42 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "tests/ols-docker"] + path = tests/ols-docker + url = https://github.com/Ensembl/OLS-docker + branch = features/ols4 diff --git a/.travis.yml b/.travis.yml index 8c26c8f..986b612 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,11 @@ python: services: - docker before_install: - - docker run --pull=always -it --privileged ensemblorg/ensembl-ols-docker:latest -p 8080:8080 + - git submodule sync + - git submodule update --init --recursive --remote + - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ + - export OLS4_CONFIG=./dataload/configs/ols-config.json + - docker compose -f ./tests/ols-docker/docker-compose.yml up -d install: - pip install -e . - pip install -r requirements-test.txt diff --git a/tests/ols-docker b/tests/ols-docker new file mode 160000 index 0000000..3df1ce5 --- /dev/null +++ b/tests/ols-docker @@ -0,0 +1 @@ +Subproject commit 3df1ce5b13bcf79291854f70d06f85a410e73435 From 5b23f47fd9d59937221ffdb0818d2553f1109d11 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:05:35 +0000 Subject: [PATCH 07/25] Added 3.8 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 986b612..28257f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "3.6" - "3.7" + - "3.8" services: - docker before_install: From 34a5c7f0e025b9a60375411f6216774c1f4b920f Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:15:05 +0000 Subject: [PATCH 08/25] Another attempt --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 28257f7..bac2774 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: - git submodule update --init --recursive --remote - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ - export OLS4_CONFIG=./dataload/configs/ols-config.json - - docker compose -f ./tests/ols-docker/docker-compose.yml up -d + - COMPOSE_FILE=./tests/ols-docker/docker-compose.yml docker compose up -d install: - pip install -e . - pip install -r requirements-test.txt From 203f072dbf8bd751175575b67e130213474fef2d Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:22:47 +0000 Subject: [PATCH 09/25] Get docker-compose --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bac2774..4a22480 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,20 @@ python: - "3.6" - "3.7" - "3.8" +env: + - DOCKER_COMPOSE_VERSION=v2.17.3 services: - docker before_install: + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin - git submodule sync - git submodule update --init --recursive --remote - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ - export OLS4_CONFIG=./dataload/configs/ols-config.json - - COMPOSE_FILE=./tests/ols-docker/docker-compose.yml docker compose up -d + - docker-compose -f ./tests/ols-docker/docker-compose.yml up -d install: - pip install -e . - pip install -r requirements-test.txt From 2d8078bb4e6bc28aaa5c26f481d79a1ac18be979 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:25:15 +0000 Subject: [PATCH 10/25] Get docker-compose --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a22480..ad12682 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: - git submodule update --init --recursive --remote - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ - export OLS4_CONFIG=./dataload/configs/ols-config.json - - docker-compose -f ./tests/ols-docker/docker-compose.yml up -d + - docker-compose -f ./tests/ols-docker/docker-compose.yml --quiet-pull up -d install: - pip install -e . - pip install -r requirements-test.txt From 6ea82aee8118d146fad06baf83306728116c2e6f Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Fri, 1 Dec 2023 20:28:31 +0000 Subject: [PATCH 11/25] quiet-pull --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ad12682..a8ccb61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: - git submodule update --init --recursive --remote - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ - export OLS4_CONFIG=./dataload/configs/ols-config.json - - docker-compose -f ./tests/ols-docker/docker-compose.yml --quiet-pull up -d + - docker-compose -f ./tests/ols-docker/docker-compose.yml up -d --quiet-pull install: - pip install -e . - pip install -r requirements-test.txt From 32116ebb2ea908a5c45aca3e76133f3ccee90a04 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Wed, 6 Dec 2023 09:48:35 +0000 Subject: [PATCH 12/25] Removed Submodule --- .gitmodules | 4 ---- tests/ols-docker | 1 - 2 files changed, 5 deletions(-) delete mode 160000 tests/ols-docker diff --git a/.gitmodules b/.gitmodules index eecda42..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "tests/ols-docker"] - path = tests/ols-docker - url = https://github.com/Ensembl/OLS-docker - branch = features/ols4 diff --git a/tests/ols-docker b/tests/ols-docker deleted file mode 160000 index 3df1ce5..0000000 --- a/tests/ols-docker +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3df1ce5b13bcf79291854f70d06f85a410e73435 From f956d5521724d29ed112433ad7f38a9ec10e9867 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Wed, 6 Dec 2023 09:56:05 +0000 Subject: [PATCH 13/25] Use ensembl-ols4 fork. --- .gitmodules | 4 ++++ .travis.yml | 9 ++++----- tests/ensembl-ols4 | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) create mode 160000 tests/ensembl-ols4 diff --git a/.gitmodules b/.gitmodules index e69de29..3425fad 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "tests/ensembl-ols4"] + path = tests/ensembl-ols4 + url = https://github.com/Ensembl/ensembl-ols4 + branch = stable diff --git a/.travis.yml b/.travis.yml index a8ccb61..68348af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,14 @@ env: services: - docker before_install: + - git submodule sync + - git submodule update --init --recursive --remote - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - - git submodule sync - - git submodule update --init --recursive --remote - - cp ./tests/ols-config.json ./tests/ols-docker/ols4/dataload/configs/ - - export OLS4_CONFIG=./dataload/configs/ols-config.json - - docker-compose -f ./tests/ols-docker/docker-compose.yml up -d --quiet-pull + - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull --no-build --no-attach ols4-solr --no-attach ols4-neo4j --no-attach ols4-backend --no-attach ols4-dataload install: - pip install -e . - pip install -r requirements-test.txt diff --git a/tests/ensembl-ols4 b/tests/ensembl-ols4 new file mode 160000 index 0000000..a2654d6 --- /dev/null +++ b/tests/ensembl-ols4 @@ -0,0 +1 @@ +Subproject commit a2654d6b1573c3325cb287eb92bbdf880ae4dc9d From a066d70eb9854fcb5c8899620de7837bed341049 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Wed, 6 Dec 2023 10:02:32 +0000 Subject: [PATCH 14/25] Remove `no-build` --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 68348af..0c55e94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull --no-build --no-attach ols4-solr --no-attach ols4-neo4j --no-attach ols4-backend --no-attach ols4-dataload + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull --no-attach ols4-solr --no-attach ols4-neo4j --no-attach ols4-backend --no-attach ols4-dataload install: - pip install -e . - pip install -r requirements-test.txt From b3731f23da059f53c3e2ce17bf5b6a1102eeac94 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 08:43:20 +0000 Subject: [PATCH 15/25] Silence totally docker-compose output --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0c55e94..04d26f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull --no-attach ols4-solr --no-attach ols4-neo4j --no-attach ols4-backend --no-attach ols4-dataload + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d > /dev/null 2>&1 install: - pip install -e . - pip install -r requirements-test.txt From ef0d06a2992885bdf199a0bdee291b5d1eeb5c86 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 08:55:38 +0000 Subject: [PATCH 16/25] Trace the last 300 lines to get a hint of failure. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 04d26f4..3d4ef28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d > /dev/null 2>&1 + - output=$(docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up | tail -n 300) && if [ $? -ne 0 ]; then echo "Errors occurred:$output"; fi install: - pip install -e . - pip install -r requirements-test.txt From 445d210d27c17d3d6452a45ef18720b0b68009ad Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 09:00:11 +0000 Subject: [PATCH 17/25] Re-added `-d` to compose command --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d4ef28..9bba23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - "3.6" - - "3.7" +# - "3.7" - "3.8" env: - DOCKER_COMPOSE_VERSION=v2.17.3 @@ -15,7 +14,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json - - output=$(docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up | tail -n 300) && if [ $? -ne 0 ]; then echo "Errors occurred:$output"; fi + - output=$(docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d | tail -n 300) && if [ $? -ne 0 ]; then echo "Errors occurred:$output"; fi install: - pip install -e . - pip install -r requirements-test.txt From e54807ed5e0eb244e00190a66dc6c33d72817a10 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 16:17:12 +0000 Subject: [PATCH 18/25] Debug Docker compose in travis --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9bba23f..8e37c4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,8 @@ services: before_install: - git submodule sync - git submodule update --init --recursive --remote - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - export OLS4_CONFIG=./dataload/configs/ensembl-test-config.json - - output=$(docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d | tail -n 300) && if [ $? -ne 0 ]; then echo "Errors occurred:$output"; fi + - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json + - docker compose version install: - pip install -e . - pip install -r requirements-test.txt From 2d79487fc371b7d327389d7f6bc3482d1a5d27ea Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 16:25:40 +0000 Subject: [PATCH 19/25] Debug Docker compose in travis --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e37c4b..152285f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,15 @@ python: - "3.8" env: - DOCKER_COMPOSE_VERSION=v2.17.3 -services: - - docker before_install: - git submodule sync - git submodule update --init --recursive --remote + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json - - docker compose version + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d 2>&1 install: - pip install -e . - pip install -r requirements-test.txt From ac87dbc372caac1e8b00e550b095e2b9774719e1 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 16:33:59 +0000 Subject: [PATCH 20/25] Debug Docker compose in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 152285f..27367e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d 2>&1 + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d ols4-dataload ols4-backend install: - pip install -e . - pip install -r requirements-test.txt From cb346b2a6462b1a9c744e8a04976c42713cc991c Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 16:42:46 +0000 Subject: [PATCH 21/25] New attempt to remove unwanted logs --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27367e4..fcf4502 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,8 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d ols4-dataload ols4-backend + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml pull --quiet + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d install: - pip install -e . - pip install -r requirements-test.txt From bc5763e87a1f3e1a1d89935619256cb4694f62e5 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Mon, 8 Jan 2024 16:56:19 +0000 Subject: [PATCH 22/25] New attempt to remove unwanted logs... --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcf4502..258d716 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ python: # - "3.7" - "3.8" env: - - DOCKER_COMPOSE_VERSION=v2.17.3 + - DOCKER_COMPOSE_VERSION=v2.23.3 before_install: - git submodule sync - git submodule update --init --recursive --remote @@ -12,8 +12,7 @@ before_install: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml pull --quiet - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d + - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull install: - pip install -e . - pip install -r requirements-test.txt From fecc386f37322adc471841114fb4bfc535c759e3 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Tue, 16 Jan 2024 08:38:15 +0000 Subject: [PATCH 23/25] Removed submodule - will use our container now --- .gitmodules | 4 ---- tests/ensembl-ols4 | 1 - 2 files changed, 5 deletions(-) delete mode 100644 .gitmodules delete mode 160000 tests/ensembl-ols4 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 3425fad..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "tests/ensembl-ols4"] - path = tests/ensembl-ols4 - url = https://github.com/Ensembl/ensembl-ols4 - branch = stable diff --git a/tests/ensembl-ols4 b/tests/ensembl-ols4 deleted file mode 160000 index a2654d6..0000000 --- a/tests/ensembl-ols4 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a2654d6b1573c3325cb287eb92bbdf880ae4dc9d From a73c1a3ae8ab80eaebc210486a893c4f20ca9ecb Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Wed, 17 Jan 2024 09:49:08 +0000 Subject: [PATCH 24/25] Updated travis to use the latest new image --- .travis.yml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 258d716..92c8c63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,20 @@ language: python python: -# - "3.7" - "3.8" -env: - - DOCKER_COMPOSE_VERSION=v2.23.3 +services: + - docker before_install: - - git submodule sync - - git submodule update --init --recursive --remote - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - export OLS4_CONFIG=dataload/configs/ensembl-test-config.json - - docker-compose -f ./tests/ensembl-ols4/docker-compose.yml up -d --quiet-pull + - docker run -it -d -v /var/run/docker.sock:/var/run/docker.sock ensemblorg/ensembl-ols-docker:latest install: - pip install -e . - pip install -r requirements-test.txt script: + - | + # Wait for the ols4-backend to be up + until [ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8080/api)" -eq "200" ]; do + echo 'Waiting for the service to be available...' + sleep 10 + done - nosetests tests.test_basic --verbose --with-coverage --cover-package=ebi.ols after_success: - coveralls From 09b01812d1289ee383fbabb2ffd7412671bf3c20 Mon Sep 17 00:00:00 2001 From: Marc Chakiachvili Date: Wed, 17 Jan 2024 17:54:50 +0000 Subject: [PATCH 25/25] First pass on fixing tests against reduced set of Ontologies. --- .travis.yml | 4 +- ebi/ols/api/base.py | 1 + ebi/ols/api/helpers.py | 3 +- requirements.txt | 1 + tests/ols-config.json | 92 ------------------------------------------ tests/test_basic.py | 80 ++++++++++++++++++------------------ 6 files changed, 46 insertions(+), 135 deletions(-) delete mode 100644 tests/ols-config.json diff --git a/.travis.yml b/.travis.yml index 92c8c63..d7bd084 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,13 @@ script: # Wait for the ols4-backend to be up until [ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8080/api)" -eq "200" ]; do echo 'Waiting for the service to be available...' - sleep 10 + sleep 30 done - nosetests tests.test_basic --verbose --with-coverage --cover-package=ebi.ols after_success: - coveralls +after_script: + - docker stop $(docker ps -q) notifications: email: on_success: change diff --git a/ebi/ols/api/base.py b/ebi/ols/api/base.py index 291e19d..21e1ffb 100644 --- a/ebi/ols/api/base.py +++ b/ebi/ols/api/base.py @@ -549,6 +549,7 @@ def pages(self): return self._get_pages() def _get_data(self, path, document): + logger.debug(f"Getting data {path} {document}") return document.data[path]['docs'] @property diff --git a/ebi/ols/api/helpers.py b/ebi/ols/api/helpers.py index 98c1442..2c6a877 100644 --- a/ebi/ols/api/helpers.py +++ b/ebi/ols/api/helpers.py @@ -173,7 +173,8 @@ class OntologyConfig(OLSHelper): skos = None def __init__(self, **kwargs): - annotations = OntologyAnnotation(**kwargs.pop("annotations", {})) + kwargs.pop("annotations", {}) + annotations = OntologyAnnotation(**kwargs) super().__init__(annotations=annotations, **kwargs) def __repr__(self): diff --git a/requirements.txt b/requirements.txt index dadf60e..fda8bcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ inflection~=0.3.1 coreapi~=2.3.3 hal_codec~=1.0.2 requests~=2.22 +urllib3==1.26.6 diff --git a/tests/ols-config.json b/tests/ols-config.json deleted file mode 100644 index 3c1953f..0000000 --- a/tests/ols-config.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "ontologies": [ - { - "activity_status": "inactive", - "build": { - "method": "owl2obo", - "source_url": "http://purl.obolibrary.org/obo/aero.owl" - }, - "contact": { - "email": "mcourtot@gmail.com", - "label": "Melanie Courtot", - "orcid": "0000-0002-9551-6370" - }, - "description": "The Adverse Event Reporting Ontology (AERO) is an ontology aimed at supporting clinicians at the time of data entry, increasing quality and accuracy of reported adverse events", - "domain": "health", - "homepage": "http://purl.obolibrary.org/obo/aero", - "id": "aero", - "is_obsolete": true, - "layout": "ontology_detail", - "license": { - "label": "CC BY 3.0", - "logo": "http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png", - "url": "http://creativecommons.org/licenses/by/3.0/" - }, - "products": [ - { - "id": "aero.owl", - "ontology_purl": "http://purl.obolibrary.org/obo/aero.owl" - } - ], - "title": "Adverse Event Reporting Ontology" - }, - { - "activity_status": "active", - "browsers": [ - { - "label": "BioPortal", - "title": "BioPortal Browser", - "url": "http://bioportal.bioontology.org/ontologies/BFO?p=classes" - } - ], - "contact": { - "email": "phismith@buffalo.edu", - "github": "phismith", - "label": "Barry Smith", - "orcid": "0000-0003-1384-116X" - }, - "depicted_by": "https://avatars2.githubusercontent.com/u/12972134?v=3&s=200", - "description": "The upper level ontology upon which OBO Foundry ontologies are built.", - "domain": "upper", - "homepage": "http://ifomis.org/bfo/", - "id": "bfo", - "in_foundry_order": 1, - "layout": "ontology_detail", - "license": { - "label": "CC BY 4.0", - "logo": "http://mirrors.creativecommons.org/presskit/buttons/80x15/png/by.png", - "url": "http://creativecommons.org/licenses/by/4.0/" - }, - "mailing_list": "https://groups.google.com/forum/#!forum/bfo-discuss", - "ontology_purl": "http://purl.obolibrary.org/obo/bfo.owl", - "preferredPrefix": "BFO", - "products": [ - { - "id": "bfo.owl", - "ontology_purl": "http://purl.obolibrary.org/obo/bfo.owl" - }, - { - "id": "bfo.obo", - "ontology_purl": "http://purl.obolibrary.org/obo/bfo.obo" - } - ], - "repository": "https://github.com/BFO-ontology/BFO", - "review": { - "date": 2016, - "document": { - "label": "PDF", - "link": "https://drive.google.com/open?id=0B81h9ah4tAM_RnNTRUZnVGRyWXM" - } - }, - "title": "Basic Formal Ontology", - "tracker": "https://github.com/BFO-ontology/BFO/issues", - "usages": [ - { - "description": "BFO is imported by multiple OBO ontologies to standardize upper level structure", - "type": "owl_import", - "user": "http://obofoundry.org" - } - ] - } - ] -} diff --git a/tests/test_basic.py b/tests/test_basic.py index 10fd15a..ae4cae5 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -116,7 +116,7 @@ def test_ontology_terms(self): ontology = self.client.ontology("aero") terms = ontology.terms() self.assertEqual(terms.index, 0) - self.assertGreaterEqual(len(terms), ontology.number_of_terms) + self.assertGreaterEqual(ontology.number_of_terms, len(terms)) self._checkTerms(terms) def test_ontology_individuals(self): @@ -128,7 +128,7 @@ def test_ontology_individuals(self): def test_ontology_properties(self): ontology = self.client.ontology("aero") properties = ontology.properties() - self.assertGreaterEqual(len(properties), ontology.number_of_properties) + self.assertGreaterEqual(ontology.number_of_properties, len(properties)) self._checkProperties(properties) def test_list_range(self): @@ -157,24 +157,23 @@ def test_list_filters(self): Test ontology terms api filtering options :return: """ - filters = {'short_form': 'DUO_0000024'} - ontology = self.client.ontology('duo') - ontologies = self.client.ontologies(filters={'fake_filter': 'fake_value'}) + filters = {'short_form': 'BFO_0000030"'} + ontology = self.client.ontology('bfo') terms = ontology.terms(filters=filters) for term in terms: - self.assertEqual(term.short_form, 'DUO_0000024') - filters = {'obo_id': 'DUO:0000024'} + self.assertEqual(term.short_form, 'BFO_0000030') + filters = {'obo_id': 'BFO:0000030'} terms = ontology.terms(filters=filters) for term in terms: - self.assertEqual(term.obo_id, 'DUO:0000024') + self.assertEqual(term.obo_id, 'BFO:0000030') - filters = {'iri': 'http://purl.obolibrary.org/obo/DUO_0000017'} + filters = {'iri': 'http://purl.obolibrary.org/obo/BFO_0000034'} terms = ontology.terms(filters=filters) for term in terms: - self.assertEqual(term.short_form, 'DUO_0000017') - self.assertEqual(term.obo_id, 'DUO:0000017') - self.assertEqual(term.iri, 'http://purl.obolibrary.org/obo/DUO_0000017') + self.assertEqual(term.short_form, 'BFO_0000034') + self.assertEqual(term.obo_id, 'BFO:0000034') + self.assertEqual(term.iri, 'http://purl.obolibrary.org/obo/BFO_0000034') def test_terms(self): """ @@ -182,10 +181,13 @@ def test_terms(self): Should warn that test may be long according to the nnumber of terms involved :return: """ - term_1 = helpers.Term(ontology_name='duo', iri='http://purl.obolibrary.org/obo/DUO_0000026') + term_1 = helpers.Term(ontology_name='bfo', iri='http://purl.obolibrary.org/obo/BFO_0000028') ancestors = term_1.load_relation('ancestors') + ancestors_obo_ids = [] for ancestor in ancestors: + ancestors_obo_ids.append(ancestors.obo_id) self._checkTerm(ancestor) + self.assertTrue('BFO:0000004' in ancestors_obo_ids) self._checkTerm(term_1) def test_dynamic_links(self): @@ -231,7 +233,7 @@ def test_search_filters(self): def test_childrenOf_is_valid_response_filter(self): # Given - filters = {'childrenOf':'A'} + filters = {'childrenOf': 'A'} expected_filters = filters.copy() # When actual_filters = self.client.search.filters_response(filters) @@ -240,7 +242,7 @@ def test_childrenOf_is_valid_response_filter(self): def test_childrenOf_value_can_be_set(self): # Given - filters = {'childrenOf':{'A','B'}} + filters = {'childrenOf': {'A', 'B'}} expected_filters = filters.copy() # When actual_filters = self.client.search.filters_response(filters) @@ -257,24 +259,24 @@ def test_childrenOf_value_cannot_be_list(self): def test_single_childrenOf_filters_search_results(self): # Given query = 'chemical' - ontology = 'phi' + ontology = 'bfo' filters = { 'fieldList': {'iri'}, - 'childrenOf': 'http://purl.obolibrary.org/obo/PHI_1000000' - } + 'childrenOf': 'http://purl.obolibrary.org/obo/BFO_123456' + } # When actual_terms = self.client.search( - query=query, + query=query, ontology=ontology, filters=filters) - actual_iris = { term.iri for term in actual_terms } + actual_iris = {term.iri for term in actual_terms} # Then expected_iris = { - 'http://purl.obolibrary.org/obo/PHI_1000003', - 'http://purl.obolibrary.org/obo/PHI_1000002' + 'http://purl.obolibrary.org/obo/PHI_1000003', + 'http://purl.obolibrary.org/obo/PHI_1000002' } self.assertEqual(actual_iris, expected_iris) - + def test_multiple_childrenOf_filters_search_results(self): # Given query = 'chemical' @@ -288,15 +290,15 @@ def test_multiple_childrenOf_filters_search_results(self): } # When actual_terms = self.client.search( - query=query, + query=query, ontology=ontology, filters=filters) - actual_iris = { term.iri for term in actual_terms } + actual_iris = {term.iri for term in actual_terms} # Then expected_iris = { - 'http://purl.obolibrary.org/obo/PHI_1000003', - 'http://purl.obolibrary.org/obo/PHI_1000002', - 'http://purl.obolibrary.org/obo/PHI_2000004' + 'http://purl.obolibrary.org/obo/PHI_1000003', + 'http://purl.obolibrary.org/obo/PHI_1000002', + 'http://purl.obolibrary.org/obo/PHI_2000004' } self.assertEqual(actual_iris, expected_iris) @@ -310,10 +312,10 @@ def test_childrenOf_non_existant_parent_yields_no_search_results(self): } # When actual_terms = self.client.search( - query=query, + query=query, ontology=ontology, filters=filters) - actual_iris = { term.iri for term in actual_terms } + actual_iris = {term.iri for term in actual_terms} # Then expected_iris = set() self.assertEqual(actual_iris, expected_iris) @@ -322,7 +324,7 @@ def test_search_kwargs(self): """ Test Search feature : - kwargs passed """ - mixed = self.client.search(query='go', type='property') + mixed = self.client.search(query='bfo', type='property') self.assertGreaterEqual(len(mixed), 15) clazz = [] @@ -439,23 +441,20 @@ def test_namespace(self): self.assertEqual(h_term.namespace, 'aero') # retrieved from obo_name_space annotation - h_term = helpers.Term(ontology_name='duo', iri='http://purl.obolibrary.org/obo/DUO_0000017') + h_term = helpers.Term(ontology_name='bfo', iri='http://purl.obolibrary.org/obo/BFO_0000034') self.client.detail(h_term) - self.assertEqual(h_term.namespace, 'duo') + self.assertEqual(h_term.namespace, 'bfo') def test_term_description(self): h_term = self.client.detail(iri="http://www.w3.org/2002/07/owl#Thing", - ontology_name='duo', type=helpers.Term) + ontology_name='aero', type=helpers.Term) self.assertEqual('', h_term.description) def test_properties_retrieval(self): subsets = "is quality measurement of" - - s_subsets = self.client.search(query=subsets, ontology='aero', type='property', - exact='true') - seen = set() - self.assertEqual(len(s_subsets), 1) - subset = s_subsets[0] + s_subsets = self.client.search(query=subsets, ontology='aero', type='property', exact='true') + self.assertEqual(len(s_subsets), 2) + subset = s_subsets[1] d_subset = self.client.property(identifier=subset.iri) self.assertEqual(d_subset.definition, subset.definition) self.assertEqual(d_subset.accession, 'IAO:0000221') @@ -463,7 +462,6 @@ def test_properties_retrieval(self): def test_term_definition(self): o_term = self.client.detail(iri="http://purl.obolibrary.org/obo/BFO_0000015", ontology_name='bfo', type=helpers.Term) - self.assertEqual(o_term.description, o_term.obo_definition_citation[0]['definition']) self.assertEqual(o_term.description, 'p is a process = Def. p is an occurrent that has temporal proper parts ' 'and for some time t, p s-depends_on some material entity at t. (axiom ' 'label in BFO2 Reference: [083-003])')