diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml deleted file mode 100644 index d8eadf4f0..000000000 --- a/.github/workflows/ruby-unit-tests.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Ruby Unit Tests - -on: - push: - branches: - - '**' - tags-ignore: - - '**' # ignore all tag pushes - pull_request: - -jobs: - test: - strategy: - fail-fast: false - matrix: - backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up solr configsets - run: ./test/solr/generate_ncbo_configsets.sh - - name: create config.rb file - run: cp config/config.test.rb config/config.rb - - name: Build docker compose - run: docker compose --profile 4store build # profile flag is set in order to build all containers in this step - - name: Run unit tests - # unit tests are run inside a container - # http://docs.codecov.io/docs/testing-with-docker - run: | - ci_env=`bash <(curl -s https://codecov.io/env)` - docker compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: unittests - verbose: true - fail_ci_if_error: false # optional (default = false) diff --git a/.github/workflows/testkit-unit-tests.yml b/.github/workflows/testkit-unit-tests.yml new file mode 100644 index 000000000..358194a41 --- /dev/null +++ b/.github/workflows/testkit-unit-tests.yml @@ -0,0 +1,87 @@ +name: Docker Unit Tests + +on: + push: + branches: + - '**' + tags-ignore: + - '**' + pull_request: + +env: + # CI execution mode for backend tests: + # - container: run `test:docker::container` (default) + # - native: run `test:docker:` on host Ruby + OPTK_CI_RUN_MODE: ${{ vars.OPTK_CI_RUN_MODE || 'container' }} + # Example override to force native mode in this workflow file: + # OPTK_CI_RUN_MODE: native + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + backends: ${{ steps.cfg.outputs.backends }} + steps: + - uses: actions/checkout@v4 + + - id: cfg + name: Read backend matrix from .ontoportal-testkit.yml + run: | + BACKENDS=$(ruby -ryaml -rjson -e 'c=YAML.safe_load_file(".ontoportal-testkit.yml") || {}; b=c["backends"] || %w[fs ag vo gd]; puts JSON.generate(b)') + echo "backends=$BACKENDS" >> "$GITHUB_OUTPUT" + + test: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + backend: ${{ fromJson(needs.prepare.outputs.backends) }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Ruby from .ruby-version + uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - name: Set up Java 11 (native mode) + if: env.OPTK_CI_RUN_MODE == 'native' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Install native system dependencies + if: env.OPTK_CI_RUN_MODE == 'native' + run: | + sudo apt-get update + sudo apt-get install -y raptor2-utils + + - name: Run unit tests + env: + CI: "true" + TESTOPTS: "--verbose" + BACKEND: ${{ matrix.backend }} + run: | + MODE="${OPTK_CI_RUN_MODE:-container}" + TASK="test:docker:${BACKEND}" + if [ "$MODE" = "container" ]; then + TASK="${TASK}:container" + elif [ "$MODE" != "native" ]; then + echo "Invalid OPTK_CI_RUN_MODE=$MODE (expected container or native)" + exit 1 + fi + + bundle exec rake "$TASK" + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: unittests,${{ matrix.backend }} + verbose: true + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index e9cd5d0d2..ff8778f94 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.swp *.gemtags .bundle +.serena *.tags* .DS_Store diff --git a/.ontoportal-testkit.yml b/.ontoportal-testkit.yml new file mode 100644 index 000000000..ded529258 --- /dev/null +++ b/.ontoportal-testkit.yml @@ -0,0 +1,8 @@ +component_name: ontologies_linked_data +app_service: test-container +backends: + - fs + - ag + - vo + - gd +dependency_services: {} diff --git a/.ruby-version b/.ruby-version index 9cec7165a..f15386a5d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.1.6 +3.2.10 diff --git a/Dockerfile b/Dockerfile index b9529fd14..7946d8bf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,21 @@ -ARG RUBY_VERSION=3.1 +ARG RUBY_VERSION=3.2 ARG DISTRO=bullseye - -FROM ruby:$RUBY_VERSION-$DISTRO - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - git \ - libxml2 \ - libxslt-dev \ - openjdk-11-jre-headless \ - raptor2-utils \ - && rm -rf /var/lib/apt/lists/* +ARG TESTKIT_BASE_IMAGE=ontoportal/testkit-base:ruby${RUBY_VERSION}-${DISTRO} +FROM ${TESTKIT_BASE_IMAGE} WORKDIR /app -COPY Gemfile* *.gemspec ./ - -# Copy only the `version.rb` file to prevent missing file errors! +# Copy only the `version.rb` file to prevent missing file errors COPY lib/ontologies_linked_data/version.rb lib/ontologies_linked_data/ -#Install the exact Bundler version from Gemfile.lock (if it exists) -RUN gem update --system && \ - if [ -f Gemfile.lock ]; then \ +COPY Gemfile* *.gemspec ./ + +# Respect the project's Bundler lock when present. +RUN if [ -f Gemfile.lock ]; then \ BUNDLER_VERSION=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | tr -d ' '); \ gem install bundler -v "$BUNDLER_VERSION"; \ - else \ - gem install bundler; \ fi -RUN bundle config set --global no-document 'true' RUN bundle install --jobs 4 --retry 3 COPY . ./ diff --git a/Gemfile b/Gemfile index 95414ea2f..3408d6051 100644 --- a/Gemfile +++ b/Gemfile @@ -2,10 +2,9 @@ source 'https://rubygems.org' gemspec -gem 'activesupport', '~> 4' +gem 'activesupport' gem 'addressable', '~> 2.8' gem 'bcrypt', '~> 3.0' -gem 'cube-ruby', require: 'cube' gem 'ffi' gem 'libxml-ruby' gem 'multi_json', '~> 1.0' @@ -13,32 +12,36 @@ gem 'oj', '~> 3.0' gem 'omni_logger' gem 'pony' gem 'rack' -gem 'rake', '~> 10.0' +gem 'rake' gem 'request_store' gem 'rest-client' gem 'rsolr' -gem 'thin', '~> 1.0' # compatibility version pin. thin should be replaced with webmoc -gem "down", "~> 5.0" +gem 'jwt' +gem 'json-ld', '~> 3.2.0' # Testing group :test do gem 'email_spec' - gem 'minitest', '~> 4' - gem 'minitest-reporters', '>= 0.5.0' + gem 'minitest' + gem 'minitest-reporters' gem 'mocha', '~> 2.7' gem 'mock_redis', '~> 0.5' + gem 'ontoportal_testkit', github: 'alexskr/ontoportal_testkit', branch: 'main' gem 'pry' gem 'rack-test', '~> 0.6' gem 'simplecov' gem 'simplecov-cobertura' # for codecov.io + gem "thin", "~> 1.8.2" + gem 'webmock' end group :development do gem 'rubocop', require: false end # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', github: 'ncbo/goo', branch: 'develop' -gem 'sparql-client', github: 'ncbo/sparql-client', tag: 'v6.3.0' +gem 'goo', github: 'ncbo/goo', branch: 'main' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'main' + gem 'public_suffix', '~> 5.1.1' gem 'net-imap', '~> 0.4.18' diff --git a/Gemfile.lock b/Gemfile.lock index 815b5cbff..e2f199c81 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,14 +1,24 @@ +GIT + remote: https://github.com/alexskr/ontoportal_testkit.git + revision: 9944d1d5f57b11e8ae9ccda11a83c7c5b2348870 + branch: main + specs: + ontoportal_testkit (0.1.0) + rake (>= 13.0) + GIT remote: https://github.com/ncbo/goo.git - revision: ca5f9d858eef89923903236fe6f76c78271e538d - branch: develop + revision: 2411deb0143c773b9e6d89b3232c9cca0d50e2bd + branch: main specs: goo (0.0.2) addressable (~> 2.8) pry - rdf (= 1.0.8) + rdf + rdf-raptor + rdf-rdfxml + rdf-vocab redis - request_store rest-client rsolr sparql-client @@ -16,13 +26,12 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: 512edc320b43e83971835dc046b4923485e8f70e - tag: v6.3.0 + revision: fa69937120104bafb3d3b8350e4f2df2efb7c247 + branch: main specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) + sparql-client (3.2.2) + net-http-persistent (~> 4.0, >= 4.0.2) + rdf (~> 3.2, >= 3.2.11) PATH remote: . @@ -30,6 +39,7 @@ PATH ontologies_linked_data (0.0.1) activesupport bcrypt + down (~> 5.0) goo json libxml-ruby @@ -46,93 +56,126 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (4.0.13) - i18n (~> 0.6, >= 0.6.9) - minitest (~> 4.2) - multi_json (~> 1.3) - thread_safe (~> 0.1) - tzinfo (~> 0.3.37) - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + activesupport (8.1.2) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + json + logger (>= 1.4.2) + minitest (>= 5.1) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) + addressable (2.8.9) + public_suffix (>= 2.0.2, < 8.0) ansi (1.5.0) ast (2.4.3) - bcrypt (3.1.20) - bigdecimal (3.2.2) + base64 (0.3.0) + bcp47_spec (0.2.1) + bcrypt (3.1.21) + bigdecimal (3.3.1) builder (3.3.0) childprocess (5.1.0) logger (~> 1.5) coderay (1.1.3) - concurrent-ruby (1.3.5) - connection_pool (2.5.3) - cube-ruby (0.0.3) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) + crack (1.0.1) + bigdecimal + rexml daemons (1.4.1) - date (3.4.1) + date (3.5.1) docile (1.4.1) domain_name (0.6.20240107) down (5.4.2) addressable (~> 2.8) + drb (2.2.3) email_spec (2.3.0) htmlentities (~> 4.3.3) launchy (>= 2.1, < 4.0) mail (~> 2.7) eventmachine (1.2.7) - faraday (2.13.4) + faraday (2.14.1) faraday-net_http (>= 2.0, < 3.5) json logger - faraday-net_http (3.4.1) - net-http (>= 0.5.0) - ffi (1.17.2-aarch64-linux-gnu) - ffi (1.17.2-arm64-darwin) - ffi (1.17.2-x86_64-linux-gnu) - hashie (5.0.0) + faraday-net_http (3.4.2) + net-http (~> 0.5) + ffi (1.17.3-aarch64-linux-gnu) + ffi (1.17.3-arm64-darwin) + ffi (1.17.3-x86_64-darwin) + ffi (1.17.3-x86_64-linux-gnu) + hashdiff (1.2.1) htmlentities (4.3.4) http-accept (1.7.0) - http-cookie (1.0.8) + http-cookie (1.1.0) domain_name (~> 0.5) - i18n (0.9.5) + i18n (1.14.8) concurrent-ruby (~> 1.0) - json (2.13.2) - json_pure (2.8.1) + io-console (0.8.2) + json (2.19.2) + json-canonicalization (0.4.0) + json-ld (3.2.5) + htmlentities (~> 4.3) + json-canonicalization (~> 0.3, >= 0.3.2) + link_header (~> 0.0, >= 0.0.8) + multi_json (~> 1.15) + rack (>= 2.2, < 4) + rdf (~> 3.2, >= 3.2.10) + json-schema (6.1.0) + addressable (~> 2.8) + bigdecimal (>= 3.1, < 5) + jwt (3.1.2) + base64 language_server-protocol (3.17.0.5) launchy (3.1.1) addressable (~> 2.8) childprocess (~> 5.0) logger (~> 1.6) libxml-ruby (5.0.5) + link_header (0.0.8) lint_roller (1.1.0) logger (1.7.0) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.8.1) + mail (2.9.0) + logger mini_mime (>= 0.1.1) net-imap net-pop net-smtp + mcp (0.8.0) + json-schema (>= 4.1) method_source (1.1.0) mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2025.0826) + mime-types-data (3.2026.0317) mini_mime (1.1.5) - minitest (4.7.5) - minitest-reporters (0.14.24) + minitest (6.0.2) + drb (~> 2.0) + prism (~> 1.5) + minitest-reporters (1.7.1) ansi builder - minitest (>= 2.12, < 5.0) - powerbar - mocha (2.7.1) + minitest (>= 5.0) + ruby-progressbar + mocha (2.8.2) ruby2_keywords (>= 0.0.5) - mock_redis (0.51.0) + mock_redis (0.53.0) redis (~> 5) - multi_json (1.17.0) - net-ftp (0.3.8) + multi_json (1.19.1) + net-ftp (0.3.9) net-protocol time - net-http (0.6.0) - uri - net-http-persistent (2.9.4) - net-imap (0.4.22) + net-http (0.9.1) + uri (>= 0.11.1) + net-http-persistent (4.0.8) + connection_pool (>= 2.2.4, < 4) + net-imap (0.4.23) date net-protocol net-pop (0.1.2) @@ -142,38 +185,59 @@ GEM net-smtp (0.5.1) net-protocol netrc (0.11.0) - oj (3.16.11) + oj (3.16.15) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) logger ostruct (0.6.3) parallel (1.27.0) - parser (3.3.9.0) + parser (3.3.10.2) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) - powerbar (2.0.1) - hashie (>= 1.1.0) - prism (1.4.0) - pry (0.15.2) + prism (1.9.0) + pry (0.16.0) coderay (~> 1.1) method_source (~> 1.0) + reline (>= 0.6.0) public_suffix (5.1.1) racc (1.8.1) - rack (2.2.17) + rack (2.2.22) rack-test (0.8.3) rack (>= 1.0, < 3) rainbow (3.1.1) - rake (10.5.0) - rdf (1.0.8) - addressable (>= 2.2) + rake (13.3.1) + rdf (3.3.4) + bcp47_spec (~> 0.2) + bigdecimal (~> 3.1, >= 3.1.5) + link_header (~> 0.0, >= 0.0.8) + logger (~> 1.5) + ostruct (~> 0.6) + readline (~> 0.0) + rdf-raptor (3.2.0) + ffi (~> 1.15) + rdf (~> 3.2) + rdf-rdfxml (3.3.0) + builder (~> 3.2, >= 3.2.4) + htmlentities (~> 4.3) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + rdf-vocab (3.3.3) + rdf (~> 3.3) + rdf-xsd (3.3.0) + rdf (~> 3.3) + rexml (~> 3.2) + readline (0.0.4) + reline redis (5.4.1) redis-client (>= 0.22.0) - redis-client (0.25.2) + redis-client (0.28.0) connection_pool - regexp_parser (2.11.2) + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) @@ -181,32 +245,34 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.4.1) + rexml (3.4.4) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.79.2) + rubocop (1.85.1) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) + mcp (~> 0.6) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.46.0, < 2.0) + rubocop-ast (>= 1.49.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.46.0) + rubocop-ast (1.49.0) parser (>= 3.3.7.2) - prism (~> 1.4) + prism (~> 1.7) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) - rubyzip (3.0.2) + rubyzip (3.2.2) + securerandom (0.4.1) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-cobertura (3.0.0) + simplecov-cobertura (3.1.0) rexml simplecov (~> 0.19) simplecov-html (0.13.2) @@ -216,37 +282,40 @@ GEM daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) rack (>= 1, < 3) - thread_safe (0.3.6) - time (0.4.1) + time (0.4.2) date - timeout (0.4.3) - tzinfo (0.3.62) - unicode-display_width (3.1.5) - unicode-emoji (~> 4.0, >= 4.0.4) - unicode-emoji (4.0.4) - uri (1.0.3) + timeout (0.6.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + uri (1.1.1) uuid (2.3.9) macaddr (~> 1.0) + webmock (3.26.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) PLATFORMS - aarch64-linux - arm64-darwin-22 - arm64-darwin-23 - arm64-darwin-24 - x86_64-linux + aarch64-linux-gnu + arm64-darwin + x86_64-darwin + x86_64-linux-gnu DEPENDENCIES - activesupport (~> 4) + activesupport addressable (~> 2.8) bcrypt (~> 3.0) - cube-ruby - down (~> 5.0) email_spec ffi goo! + json-ld (~> 3.2.0) + jwt libxml-ruby - minitest (~> 4) - minitest-reporters (>= 0.5.0) + minitest + minitest-reporters mocha (~> 2.7) mock_redis (~> 0.5) multi_json (~> 1.0) @@ -254,12 +323,13 @@ DEPENDENCIES oj (~> 3.0) omni_logger ontologies_linked_data! + ontoportal_testkit! pony pry public_suffix (~> 5.1.1) rack rack-test (~> 0.6) - rake (~> 10.0) + rake request_store rest-client rsolr @@ -267,7 +337,127 @@ DEPENDENCIES simplecov simplecov-cobertura sparql-client! - thin (~> 1.0) + thin (~> 1.8.2) + webmock + +CHECKSUMS + activesupport (8.1.2) sha256=88842578ccd0d40f658289b0e8c842acfe9af751afee2e0744a7873f50b6fdae + addressable (2.8.9) sha256=cc154fcbe689711808a43601dee7b980238ce54368d23e127421753e46895485 + ansi (1.5.0) sha256=5408253274e33d9d27d4a98c46d2998266fd51cba58a7eb9d08f50e57ed23592 + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + bcp47_spec (0.2.1) sha256=3fd62edf96c126bd9624e4319ac74082a966081859d1ee0ef3c3041640a37810 + bcrypt (3.1.21) sha256=5964613d750a42c7ee5dc61f7b9336fb6caca429ba4ac9f2011609946e4a2dcf + bigdecimal (3.3.1) sha256=eaa01e228be54c4f9f53bf3cc34fe3d5e845c31963e7fcc5bedb05a4e7d52218 + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + childprocess (5.1.0) sha256=9a8d484be2fd4096a0e90a0cd3e449a05bc3aa33f8ac9e4d6dcef6ac1455b6ec + coderay (1.1.3) sha256=dc530018a4684512f8f38143cd2a096c9f02a1fc2459edcfe534787a7fc77d4b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a + crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e + daemons (1.4.1) sha256=8fc76d76faec669feb5e455d72f35bd4c46dc6735e28c420afb822fac1fa9a1d + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e + domain_name (0.6.20240107) sha256=5f693b2215708476517479bf2b3802e49068ad82167bcd2286f899536a17d933 + down (5.4.2) sha256=516e5e01e7a96214a7e2cd155aac6f700593038ae6c857c0f4a05413b1c58acf + drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373 + email_spec (2.3.0) sha256=df23be7a131186f7a3d5be3b35eaac9196f9ac13bd26c9c3d59341e13d852d11 + eventmachine (1.2.7) sha256=994016e42aa041477ba9cff45cbe50de2047f25dd418eba003e84f0d16560972 + faraday (2.14.1) sha256=a43cceedc1e39d188f4d2cdd360a8aaa6a11da0c407052e426ba8d3fb42ef61c + faraday-net_http (3.4.2) sha256=f147758260d3526939bf57ecf911682f94926a3666502e24c69992765875906c + ffi (1.17.3-aarch64-linux-gnu) sha256=28ad573df26560f0aedd8a90c3371279a0b2bd0b4e834b16a2baa10bd7a97068 + ffi (1.17.3-arm64-darwin) sha256=0c690555d4cee17a7f07c04d59df39b2fba74ec440b19da1f685c6579bb0717f + ffi (1.17.3-x86_64-darwin) sha256=1f211811eb5cfaa25998322cdd92ab104bfbd26d1c4c08471599c511f2c00bb5 + ffi (1.17.3-x86_64-linux-gnu) sha256=3746b01f677aae7b16dc1acb7cb3cc17b3e35bdae7676a3f568153fb0e2c887f + goo (0.0.2) + hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 + htmlentities (4.3.4) sha256=125a73c6c9f2d1b62100b7c3c401e3624441b663762afa7fe428476435a673da + http-accept (1.7.0) sha256=c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126 + http-cookie (1.1.0) sha256=38a5e60d1527eebc396831b8c4b9455440509881219273a6c99943d29eadbb19 + i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5 + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + json (2.19.2) sha256=e7e1bd318b2c37c4ceee2444841c86539bc462e81f40d134cf97826cb14e83cf + json-canonicalization (0.4.0) sha256=73ea88b68f210d1a09c2116d4cd1ff5a39684c6a409f7ccac70d5b1a426a8bef + json-ld (3.2.5) sha256=98b96f1831b0fe9c7d2568a7d43b64f6b8c3f5892d55ccf91640e32a99c273fc + json-schema (6.1.0) sha256=6bf70a2cfb6dfd5a06da28093fa8190f324c88eabd36a7f47097f227321dc702 + jwt (3.1.2) sha256=af6991f19a6bb4060d618d9add7a66f0eeb005ac0bc017cd01f63b42e122d535 + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + launchy (3.1.1) sha256=72b847b5cc961589dde2c395af0108c86ff0119f42d4648d25b5440ebb10059e + libxml-ruby (5.0.5) sha256=f1bc07152982df555d70159a694ee2a53539de2cdad4b3c8a447fbb15e7e4e9a + link_header (0.0.8) sha256=15c65ce43b29f739b30d05e5f25c22c23797e89cf6f905dbb595fb4c70cb55f9 + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + macaddr (1.7.2) sha256=da377809968bbc1160bf02a999e916bb3255000007291d9d1a49a93ceedadf82 + mail (2.9.0) sha256=6fa6673ecd71c60c2d996260f9ee3dd387d4673b8169b502134659ece6d34941 + mcp (0.8.0) sha256=ae8bd146bb8e168852866fd26f805f52744f6326afb3211e073f78a95e0c34fb + method_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5 + mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 + mime-types-data (3.2026.0317) sha256=77f078a4d8631d52b842ba77099734b06eddb7ad339d792e746d2272b67e511b + mini_mime (1.1.5) sha256=8681b7e2e4215f2a159f9400b5816d85e9d8c6c6b491e96a12797e798f8bccef + minitest (6.0.2) sha256=db6e57956f6ecc6134683b4c87467d6dd792323c7f0eea7b93f66bd284adbc3d + minitest-reporters (1.7.1) sha256=5060413a0c95b8c32fe73e0606f3631c173a884d7900e50013e15094eb50562c + mocha (2.8.2) sha256=1f77e729db47e72b4ef776461ce20caeec2572ffdf23365b0a03608fee8f4eee + mock_redis (0.53.0) sha256=11319e134b2905a7bb33464db696587bbcd5f5a1d8fbd2237f464d29e8b1cf6a + multi_json (1.19.1) sha256=7aefeff8f2c854bf739931a238e4aea64592845e0c0395c8a7d2eea7fdd631b7 + net-ftp (0.3.9) sha256=307817ccf7f428f79d083f7e36dbb46a9d1d375e0d23027824de1866f0b13b65 + net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 + net-http-persistent (4.0.8) sha256=ef3de8319d691537b329053fae3a33195f8b070bbbfae8bf1a58c796081960e6 + net-imap (0.4.23) sha256=b86cc99340a47b1f37e721781a147882811796a92eb635562eb3145ed2f68953 + net-pop (0.1.2) sha256=848b4e982013c15b2f0382792268763b748cce91c9e91e36b0f27ed26420dff3 + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + net-smtp (0.5.1) sha256=ed96a0af63c524fceb4b29b0d352195c30d82dd916a42f03c62a3a70e5b70736 + netrc (0.11.0) sha256=de1ce33da8c99ab1d97871726cba75151113f117146becbe45aa85cb3dabee3f + oj (3.16.15) sha256=4d3324cac3e8fef54c0fa250b2af26a16dadd9f9788a1d6b1b2098b793a1b2cd + omni_logger (0.1.4) sha256=b61596f7d96aa8426929e46c3500558d33e838e1afd7f4735244feb4d082bb3e + ontologies_linked_data (0.0.1) + ontoportal_testkit (0.1.0) + ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.2) sha256=6f60c84aa4bdcedb6d1a2434b738fe8a8136807b6adc8f7f53b97da9bc4e9357 + pony (1.13.1) sha256=ab507c8ade8b35de96f1e75c0ae4566a3c40ac8a0d5101433969b6fd29c718a7 + prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85 + pry (0.16.0) sha256=d76c69065698ed1f85e717bd33d7942c38a50868f6b0673c636192b3d1b6054e + public_suffix (5.1.1) sha256=250ec74630d735194c797491c85e3c6a141d7b5d9bd0b66a3fa6268cf67066ed + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + rack (2.2.22) sha256=c5cf0b7f872559966d974abe3101a57d51caf12504ee76290b98720004f64542 + rack-test (0.8.3) sha256=2fec9a98d366017788dbcddb60430d58c4fec9733d6a695c8a216fee7f1a3292 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rdf (3.3.4) sha256=a77fa0821e5b4e2bea9fdbb7c9d980564c89d27e81979690ce5c9e6bc80859c1 + rdf-raptor (3.2.0) sha256=43033ca3992b7dbefd5c2464997bd85bdd06961432db94645d4436ffd43bf405 + rdf-rdfxml (3.3.0) sha256=11647f6111b97b6a9b82413bd9810d4bb5524aa7dd06b3c1330bf58ec3aa6a9a + rdf-vocab (3.3.3) sha256=d3b642edb37be7b37b73cafa9e01d55762f99292838e7b0868a3575bd297bf8b + rdf-xsd (3.3.0) sha256=fab51d27b20344237d9b622ef32e83e4c44940840bfc76a245ce6b6abba44772 + readline (0.0.4) sha256=6138eef17be2b98298b672c3ea63bf9cb5158d401324f26e1e84f235879c1d6a + redis (5.4.1) sha256=b5e675b57ad22b15c9bcc765d5ac26f60b675408af916d31527af9bd5a81faae + redis-client (0.28.0) sha256=888892f9cd8787a41c0ece00bdf5f556dfff7770326ce40bb2bc11f1bfec824b + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + request_store (1.7.0) sha256=e1b75d5346a315f452242a68c937ef8e48b215b9453a77a6c0acdca2934c88cb + rest-client (2.1.0) sha256=35a6400bdb14fae28596618e312776c158f7ebbb0ccad752ff4fa142bf2747e3 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + rsolr (2.6.0) sha256=4b3bcea772cac300562775c20eeddedf63a6b7516a070cb6fbde000b09cfe12b + rubocop (1.85.1) sha256=3dbcf9e961baa4c376eeeb2a03913dca5e3987033b04d38fa538aa1e7406cc77 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubyzip (3.2.2) sha256=c0ed99385f0625415c8f05bcae33fe649ed2952894a95ff8b08f26ca57ea5b3c + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5 + simplecov-cobertura (3.1.0) sha256=6d7f38aa32c965ca2174b2e5bd88cb17138eaf629518854976ac50e628925dc5 + simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246 + simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428 + sparql-client (3.2.2) + systemu (2.6.5) sha256=01f7d014b1453b28e5781e15c4d7d63fc9221c29b174b7aae5253207a75ab33e + thin (1.8.2) sha256=1c55251aba5bee7cf6936ea18b048f4d3c74ef810aa5e6906cf6edff0df6e121 + time (0.4.2) sha256=f324e498c3bde9471d45a7d18f874c27980e9867aa5cfca61bebf52262bc3dab + timeout (0.6.0) sha256=6d722ad619f96ee383a0c557ec6eb8c4ecb08af3af62098a0be5057bf00de1af + tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + uuid (2.3.9) sha256=aec0cf592053cd6e07c13c1ef94c440aba705f22eb1ee767b39631f2760124d7 + webmock (3.26.1) sha256=4f696fb57c90a827c20aadb2d4f9058bbff10f7f043bd0d4c3f58791143b1cd7 BUNDLED WITH - 2.6.3 + 4.0.7 diff --git a/Rakefile b/Rakefile index c59273fc8..ad3ce74d3 100644 --- a/Rakefile +++ b/Rakefile @@ -4,23 +4,27 @@ task default: %w[test] Rake::TestTask.new do |t| t.libs = [] + t.warning = false t.test_files = FileList['test/**/test*.rb'] end Rake::TestTask.new do |t| t.libs = [] + t.warning = false t.name = "test:models" t.test_files = FileList['test/models/test*.rb'] end Rake::TestTask.new do |t| t.libs = [] + t.warning = false t.name = "test:rack" t.test_files = FileList['test/rack/test*.rb'] end Rake::TestTask.new do |t| t.libs = [] + t.warning = false t.name = "test:serializer" t.test_files = FileList['test/serializer/test*.rb'] end diff --git a/config/config.rb.sample b/config/config.rb.sample index 9f1d6b1c0..75a6d71d4 100644 --- a/config/config.rb.sample +++ b/config/config.rb.sample @@ -1,10 +1,10 @@ LinkedData.config do |config| config.goo_port = 9000 config.goo_host = "localhost" - config.search_server_url = "http://localhost:8983/solr/term_search_core1" - config.property_search_server_url = "http://localhost:8983/solr/prop_search_core1" + config.search_server_url = "http://localhost:8983/solr" + config.property_search_server_url = "http://localhost:8983/solr" config.repository_folder = "./test/data/ontology_files/repo" - config.rest_url_prefix = "http://data.bioontology.org/" + config.rest_url_prefix = "http://localhost:9393/" config.enable_security = false config.java_max_heap_size = '10240M' #PURL server config parameters @@ -15,6 +15,25 @@ LinkedData.config do |config| config.purl_password = "" config.purl_maintainers = "" config.purl_target_url_prefix = "http://bioportal.bioontology.org" + + config.oauth_providers = { + github: { + check: :access_token, + link: 'https://api.github.com/user' + }, + keycloak: { + check: :jwt_token, + cert: 'KEYCLOAK_SECRET_KEY' + }, + orcid: { + check: :access_token, + link: 'https://pub.orcid.org/v3.0/me' + }, + google: { + check: :access_token, + link: 'https://www.googleapis.com/oauth2/v3/userinfo' + } + } end #sometimes tmp by default cannot allocate large files diff --git a/config/config.test.rb b/config/config.test.rb index 6ea50e353..ad52fbc43 100644 --- a/config/config.test.rb +++ b/config/config.test.rb @@ -5,16 +5,16 @@ # https://github.com/ncbo/ontologies_linked_data/blob/master/lib/ontologies_linked_data/config/config.rb ### -GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" -GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" -GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" -GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" -GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 -GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" -REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : "localhost" -REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 -SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" -SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" +GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : '4store' +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : '/sparql/' +GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : '/data/' +GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : '/update/' +GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 +GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : 'localhost' +REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : 'localhost' +REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : 'http://localhost:8983/solr' +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : 'http://localhost:8983/solr' LinkedData.config do |config| config.goo_backend_name = GOO_BACKEND_NAME.to_s @@ -32,4 +32,23 @@ config.search_server_url = SOLR_TERM_SEARCH_URL.to_s config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s # config.enable_notifications = false + + config.oauth_providers = { + github: { + check: :access_token, + link: 'https://api.github.com/user' + }, + keycloak: { + check: :jwt_token, + cert: 'KEYCLOAK_SECRET_KEY' + }, + orcid: { + check: :access_token, + link: 'https://pub.orcid.org/v3.0/me' + }, + google: { + check: :access_token, + link: 'https://www.googleapis.com/oauth2/v3/userinfo' + } + } end diff --git a/config/solr/property_search/enumsconfig.xml b/config/solr/property_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d31..000000000 --- a/config/solr/property_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/property_search/mapping-ISOLatin1Accent.txt b/config/solr/property_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede774258..000000000 --- a/config/solr/property_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/property_search/schema.xml b/config/solr/property_search/schema.xml deleted file mode 100644 index 20824ea65..000000000 --- a/config/solr/property_search/schema.xml +++ /dev/null @@ -1,1179 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/config/solr/property_search/solrconfig.xml b/config/solr/property_search/solrconfig.xml deleted file mode 100644 index 771a0f322..000000000 --- a/config/solr/property_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/config/solr/solr.xml b/config/solr/solr.xml deleted file mode 100644 index d9d089e4d..000000000 --- a/config/solr/solr.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - ${solr.max.booleanClauses:500000} - ${solr.sharedLib:} - ${solr.allowPaths:} - - - - ${host:} - ${solr.port.advertise:0} - ${hostContext:solr} - - ${genericCoreNodeNames:true} - - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - - - ${socketTimeout:600000} - ${connTimeout:60000} - ${solr.shardsWhitelist:} - - - - - diff --git a/config/solr/term_search/enumsconfig.xml b/config/solr/term_search/enumsconfig.xml deleted file mode 100644 index 72e7b7d31..000000000 --- a/config/solr/term_search/enumsconfig.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - ONTOLOGY - VALUE_SET_COLLECTION - - - ANNOTATION - DATATYPE - OBJECT - - \ No newline at end of file diff --git a/config/solr/term_search/mapping-ISOLatin1Accent.txt b/config/solr/term_search/mapping-ISOLatin1Accent.txt deleted file mode 100644 index ede774258..000000000 --- a/config/solr/term_search/mapping-ISOLatin1Accent.txt +++ /dev/null @@ -1,246 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Syntax: -# "source" => "target" -# "source".length() > 0 (source cannot be empty.) -# "target".length() >= 0 (target can be empty.) - -# example: -# "À" => "A" -# "\u00C0" => "A" -# "\u00C0" => "\u0041" -# "ß" => "ss" -# "\t" => " " -# "\n" => "" - -# À => A -"\u00C0" => "A" - -# Á => A -"\u00C1" => "A" - -#  => A -"\u00C2" => "A" - -# à => A -"\u00C3" => "A" - -# Ä => A -"\u00C4" => "A" - -# Å => A -"\u00C5" => "A" - -# Æ => AE -"\u00C6" => "AE" - -# Ç => C -"\u00C7" => "C" - -# È => E -"\u00C8" => "E" - -# É => E -"\u00C9" => "E" - -# Ê => E -"\u00CA" => "E" - -# Ë => E -"\u00CB" => "E" - -# Ì => I -"\u00CC" => "I" - -# Í => I -"\u00CD" => "I" - -# Î => I -"\u00CE" => "I" - -# Ï => I -"\u00CF" => "I" - -# IJ => IJ -"\u0132" => "IJ" - -# Ð => D -"\u00D0" => "D" - -# Ñ => N -"\u00D1" => "N" - -# Ò => O -"\u00D2" => "O" - -# Ó => O -"\u00D3" => "O" - -# Ô => O -"\u00D4" => "O" - -# Õ => O -"\u00D5" => "O" - -# Ö => O -"\u00D6" => "O" - -# Ø => O -"\u00D8" => "O" - -# Œ => OE -"\u0152" => "OE" - -# Þ -"\u00DE" => "TH" - -# Ù => U -"\u00D9" => "U" - -# Ú => U -"\u00DA" => "U" - -# Û => U -"\u00DB" => "U" - -# Ü => U -"\u00DC" => "U" - -# Ý => Y -"\u00DD" => "Y" - -# Ÿ => Y -"\u0178" => "Y" - -# à => a -"\u00E0" => "a" - -# á => a -"\u00E1" => "a" - -# â => a -"\u00E2" => "a" - -# ã => a -"\u00E3" => "a" - -# ä => a -"\u00E4" => "a" - -# å => a -"\u00E5" => "a" - -# æ => ae -"\u00E6" => "ae" - -# ç => c -"\u00E7" => "c" - -# è => e -"\u00E8" => "e" - -# é => e -"\u00E9" => "e" - -# ê => e -"\u00EA" => "e" - -# ë => e -"\u00EB" => "e" - -# ì => i -"\u00EC" => "i" - -# í => i -"\u00ED" => "i" - -# î => i -"\u00EE" => "i" - -# ï => i -"\u00EF" => "i" - -# ij => ij -"\u0133" => "ij" - -# ð => d -"\u00F0" => "d" - -# ñ => n -"\u00F1" => "n" - -# ò => o -"\u00F2" => "o" - -# ó => o -"\u00F3" => "o" - -# ô => o -"\u00F4" => "o" - -# õ => o -"\u00F5" => "o" - -# ö => o -"\u00F6" => "o" - -# ø => o -"\u00F8" => "o" - -# œ => oe -"\u0153" => "oe" - -# ß => ss -"\u00DF" => "ss" - -# þ => th -"\u00FE" => "th" - -# ù => u -"\u00F9" => "u" - -# ú => u -"\u00FA" => "u" - -# û => u -"\u00FB" => "u" - -# ü => u -"\u00FC" => "u" - -# ý => y -"\u00FD" => "y" - -# ÿ => y -"\u00FF" => "y" - -# ff => ff -"\uFB00" => "ff" - -# fi => fi -"\uFB01" => "fi" - -# fl => fl -"\uFB02" => "fl" - -# ffi => ffi -"\uFB03" => "ffi" - -# ffl => ffl -"\uFB04" => "ffl" - -# ſt => ft -"\uFB05" => "ft" - -# st => st -"\uFB06" => "st" diff --git a/config/solr/term_search/schema.xml b/config/solr/term_search/schema.xml deleted file mode 100644 index 73c75b31f..000000000 --- a/config/solr/term_search/schema.xml +++ /dev/null @@ -1,1224 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/config/solr/term_search/solrconfig.xml b/config/solr/term_search/solrconfig.xml deleted file mode 100644 index 771a0f322..000000000 --- a/config/solr/term_search/solrconfig.xml +++ /dev/null @@ -1,1299 +0,0 @@ - - - - - - - - - 8.8.2 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - ${solr.max.booleanClauses:500000} - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - - - - - - - - - - - - - - - - explicit - json - true - - - - - - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - false - - - terms - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z - yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z - yyyy-MM-dd HH:mm[:ss[.SSS]][z - yyyy-MM-dd HH:mm[:ss[,SSS]][z - [EEE, ]dd MMM yyyy HH:mm[:ss] z - EEEE, dd-MMM-yy HH:mm:ss z - EEE MMM ppd HH:mm:ss [z ]yyyy - - - - - java.lang.String - text_general - - *_str - 256 - - - true - - - java.lang.Boolean - booleans - - - java.util.Date - pdates - - - java.lang.Long - java.lang.Integer - plongs - - - java.lang.Number - pdoubles - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - - - - - - - - - - diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 257ed02d8..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,140 +0,0 @@ -x-app: &app - build: - context: . - args: - RUBY_VERSION: '3.1' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_ld-dev:0.0.4 - environment: &env - COVERAGE: 'true' # enable simplecov code coverage - REDIS_HOST: redis-ut - REDIS_PORT: 6379 - SOLR_TERM_SEARCH_URL: http://solr-term-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-prop-ut:8983/solr/prop_search_core1 - stdin_open: true - tty: true - command: /bin/bash - volumes: - # bundle volume for hosting gems installed by bundle; it speeds up gem install in local development - - bundle:/usr/local/bundle - - .:/app - # mount directory containing development version of the gems if you need to use 'bundle config local' - #- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal - depends_on: &depends_on - solr-prop-ut: - condition: service_healthy - solr-term-ut: - condition: service_healthy - redis-ut: - condition: service_healthy - -services: - # environment wtih 4store backend - ruby: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: 4store - GOO_PORT: 9000 - GOO_HOST: 4store-ut - GOO_PATH_QUERY: /sparql/ - GOO_PATH_DATA: /data/ - GOO_PATH_UPDATE: /update/ - profiles: - - 4store - depends_on: - <<: *depends_on - 4store-ut: - condition: service_started - - # environment with AllegroGraph backend - ruby-agraph: - <<: *app - environment: - <<: *env - GOO_BACKEND_NAME: ag - GOO_PORT: 10035 - GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/ontoportal_test - GOO_PATH_DATA: /repositories/ontoportal_test/statements - GOO_PATH_UPDATE: /repositories/ontoportal_test/statements - profiles: - - agraph - depends_on: - <<: *depends_on - agraph-ut: - condition: service_healthy - - redis-ut: - image: redis - command: ["redis-server", "--save", "", "--appendonly", "no"] - healthcheck: - test: redis-cli ping - interval: 10s - timeout: 3s - retries: 10 - - 4store-ut: - image: bde2020/4store - platform: linux/amd64 - command: > - bash -c "4s-backend-setup --segments 4 ontoportal_kb - && 4s-backend ontoportal_kb - && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" - profiles: - - 4store - - solr-term-ut: - image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - # ports: - # - "8983:8983" - command: ["solr-precreate", "term_search_core1", "/configsets/term_search"] - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 5s - interval: 10s - timeout: 5s - retries: 5 - - solr-prop-ut: - image: solr:8 - volumes: - - ./test/solr/configsets:/configsets:ro - # ports: - # - "8984:8983" - command: ["solr-precreate", "prop_search_core1", "/configsets/property_search"] - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/prop_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] - start_period: 5s - interval: 10s - timeout: 5s - retries: 5 - - agraph-ut: - image: franzinc/agraph:v8.3.1 - platform: linux/amd64 - environment: - - AGRAPH_SUPER_USER=test - - AGRAPH_SUPER_PASSWORD=xyzzy - shm_size: 1g - # ports: - # - 10035:10035 - command: > - bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create --supersede ontoportal_test - ; agtool users add anonymous - ; agtool users grant anonymous root:ontoportal_test:rw - ; tail -f /agraph/data/agraph.log" - healthcheck: - test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"] - start_period: 30s #AllegroGraph can take a loooooong time to start - interval: 20s - timeout: 10s - retries: 20 - profiles: - - agraph - -volumes: - bundle: diff --git a/lib/ontologies_linked_data.rb b/lib/ontologies_linked_data.rb index ff221e08a..1a55ecf18 100644 --- a/lib/ontologies_linked_data.rb +++ b/lib/ontologies_linked_data.rb @@ -1,4 +1,5 @@ require 'goo' +require 'active_support/core_ext/object/blank' # Make sure we're in the load path lib_dir = "#{File.dirname(__FILE__)}/../lib" @@ -25,6 +26,7 @@ require "ontologies_linked_data/monkeypatches/class" # load before object require "ontologies_linked_data/monkeypatches/object" require "ontologies_linked_data/monkeypatches/logging" +require "ontologies_linked_data/monkeypatches/rdf_raptor_ffi" require "ontologies_linked_data/sample_data/sample_data" require "ontologies_linked_data/mappings/mappings" require "ontologies_linked_data/http_cache/cacheable_resource" diff --git a/lib/ontologies_linked_data/concerns/analytics.rb b/lib/ontologies_linked_data/concerns/analytics.rb new file mode 100644 index 000000000..58853e2f1 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/analytics.rb @@ -0,0 +1,52 @@ +module LinkedData + module Concerns + module Analytics + def self.included base + base.extend ClassMethods + end + + module ClassMethods + def load_data(field_name) + @@redis ||= Redis.new(:host => LinkedData.settings.ontology_analytics_redis_host, + :port => LinkedData.settings.ontology_analytics_redis_port, + :timeout => 30) + raw_data = @@redis.get(field_name) + raw_data.nil? ? Hash.new : Marshal.load(raw_data) + end + + def analytics_redis_key + raise NotImplementedError # the class that includes it need to implement it + end + + def load_analytics_data + self.load_data(analytics_redis_key) + end + + def analytics(year = nil, month = nil) + retrieve_analytics(year, month) + end + + # A static method for retrieving Analytics for a combination of ontologies, year, month + def retrieve_analytics(year = nil, month = nil) + analytics = self.load_analytics_data + + year = year.to_s if year + month = month.to_s if month + + unless analytics.empty? + analytics.values.each do |ont_analytics| + ont_analytics.delete_if { |key, _| key != year } unless year.nil? + ont_analytics.each { |_, val| val.delete_if { |key, __| key != month } } unless month.nil? + end + # sort results by the highest traffic values + analytics = Hash[analytics.sort_by { |_, v| v[year][month] }.reverse] if year && month + end + analytics + end + end + + end + end +end + + diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb index 4a88c12a0..624c792d2 100644 --- a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb @@ -33,7 +33,7 @@ def roots_by_query(query_body, page, paged, pagesize) #needs to get cached class_ids = [] - Goo.sparql_query_client.query(root_skos, { graphs: [self.id] }).each_solution do |s| + Goo.sparql_query_client.query(root_skos, **{ graphs: [self.id] }).each_solution do |s| class_ids << s[:root] end @@ -42,7 +42,7 @@ def roots_by_query(query_body, page, paged, pagesize) def roots_by_has_top_concept(concept_schemes, page, paged, pagesize) query_body = <<-eos - ?x #{RDF::SKOS[:hasTopConcept].to_ntriples} ?root . + ?x #{RDF::Vocab::SKOS[:hasTopConcept].to_ntriples} ?root . #{concept_schemes_filter(concept_schemes)} eos roots_by_query query_body, page, paged, pagesize @@ -50,7 +50,7 @@ def roots_by_has_top_concept(concept_schemes, page, paged, pagesize) def roots_by_top_concept_of(concept_schemes, page, paged, pagesize) query_body = <<-eos - ?root #{RDF::SKOS[:topConceptOf].to_ntriples} ?x. + ?root #{RDF::Vocab::SKOS[:topConceptOf].to_ntriples} ?x. #{concept_schemes_filter(concept_schemes)} eos roots_by_query query_body, page, paged, pagesize diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/submission_metadata_extractor.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/submission_metadata_extractor.rb deleted file mode 100644 index 1a183319b..000000000 --- a/lib/ontologies_linked_data/concerns/ontology_submissions/submission_metadata_extractor.rb +++ /dev/null @@ -1,322 +0,0 @@ -module LinkedData - module Concerns - module OntologySubmission - module MetadataExtractor - - def extract_metadata(logger = nil, heavy_extraction = true, user_params = nil) - logger ||= Logger.new(STDOUT) - logger.info("Extracting metadata from the ontology submission #{self.id}") - - unless self.valid? - logger.error("Cannot extract metadata from #{self.id} because the submission is invalid") - return - end - - @submission = self - ontology_iri = extract_ontology_iri - - if ontology_iri - old_uri = @submission.uri - @submission.uri = ontology_iri - - if @submission.valid? - @submission.save - else - logger.error("The extracted submission URI #{ontology_iri} is invalid. Discarding...") - @submission.uri = old_uri - end - end - - if heavy_extraction - begin - # Extract metadata directly from the ontology - extract_ontology_metadata(logger, user_params, skip_attrs: [:uri]) - logger.info('Additional metadata extracted') - rescue StandardError => e - message = e.message + "\n\n " + e.backtrace.join("\n ") - logger.error("Error while extracting additional metadata: #{message}") - end - end - - version_exists = @submission.version - - if version_exists.to_s.strip.empty? - version_info = extract_version - - if version_info - @submission.version = version_info - - unless @submission.valid? - logger.error("The extracted submission version \"#{version_info}\" is invalid. Discarding...") - @submission.version = nil - end - end - end - - @submission.save - end - - def extract_version - query = Goo.sparql_query_client.select(:versionInfo).distinct - .from(@submission.id) - .where([RDF::URI.new('http://bioportal.bioontology.org/ontologies/versionSubject'), - RDF::URI.new("#{Goo.namespaces[:owl].to_s}versionInfo"), - :versionInfo]) - sol = query.each_solution.first || {} - sol[:versionInfo]&.to_s - end - - def extract_ontology_iri - query = Goo.sparql_query_client.select(:uri).distinct - .from(@submission.id) - .where([:uri, - RDF::URI.new("#{Goo.namespaces[:rdf].to_s}type"), - RDF::URI.new("#{Goo.namespaces[:owl].to_s}Ontology")]) - sol = query.each_solution.first || {} - RDF::URI.new(sol[:uri]) if sol[:uri] - end - - # Extract additional metadata about the ontology - # First it extracts the main metadata, then the mapped metadata - def extract_ontology_metadata(logger, user_params, skip_attrs: []) - # The commented out line below would give priority to the existing values over extracted values - # user_params = object_to_hash(@submission) unless user_params - user_params = {} if user_params.nil? || !user_params - ontology_uri = @submission.uri - logger.info("Extracting additional metadata from ontology #{ontology_uri}") - - # go through all OntologySubmission attributes. Returns symbols - LinkedData::Models::OntologySubmission.attributes(:all).each do |attr| - next if skip_attrs.include? attr - # for attribute with the :extractedMetadata setting on, and that have not been defined by the user - attr_settings = LinkedData::Models::OntologySubmission.attribute_settings(attr) - - # Check if the user provided a non-empty value for this attribute - user_provided_value = user_params.key?(attr) && !empty_value?(user_params[attr]) - # Proceed only if the attribute is marked as extractable and the user did NOT provide a value - should_extract = attr_settings[:extractedMetadata] && !user_provided_value - next unless should_extract - - # a boolean to check if a value that should be single have already been extracted - single_extracted = false - type = enforce?(attr, :list) ? :list : :string - old_value = value(attr, type) - - unless attr_settings[:namespace].nil? - property_to_extract = "#{attr_settings[:namespace].to_s}:#{attr.to_s}" - hash_results = extract_each_metadata(ontology_uri, attr, property_to_extract, logger) - single_extracted = send_value(attr, hash_results, logger) unless hash_results.empty? - end - - # extracts attribute value from metadata mappings - attr_settings[:metadataMappings] ||= [] - - attr_settings[:metadataMappings].each do |mapping| - break if single_extracted - - hash_mapping_results = extract_each_metadata(ontology_uri, attr, mapping.to_s, logger) - single_extracted = send_value(attr, hash_mapping_results, logger) unless hash_mapping_results.empty? - end - - new_value = value(attr, type) - - if empty_value?(new_value) && !empty_value?(old_value) - logger.error("Extracted attribute #{attr} with the value #{new_value} is empty. Returning to the old value: #{old_value}.") - send_value(attr, old_value, logger) - end - end - end - - def object_to_hash(obj) - obj.instance_variables.each_with_object({}) do |ivar, hash| - key = ivar.to_s.delete("@").to_sym - value = obj.instance_variable_get(ivar) - hash[key] = value - end - end - - def empty_value?(value) - value.nil? || (value.is_a?(Array) && value.empty?) || value.to_s.strip.empty? - end - - def value(attr, type) - val = @submission.send(attr.to_s) - type.eql?(:list) ? Array(val) || [] : val || '' - end - - def send_value(attr, new_value, logger) - old_val = nil - single_extracted = false - - if enforce?(attr, :list) - old_val = value(attr, :list) - old_values = old_val.dup - new_values = new_value.values - new_values = new_values.map { |v| find_or_create_agent(attr, v, logger) }.compact if enforce?(attr, :Agent) - - old_values.push(*new_values) - - @submission.send("#{attr}=", old_values.uniq) - elsif enforce?(attr, :concatenate) - # if multiple value for this attribute, then we concatenate it - # Add the concat at the very end, to easily join the content of the array - old_val = value(attr, :string) - metadata_values = old_val.split(', ') - new_values = new_value.values.map { |x| x.to_s.split(', ') }.flatten - - @submission.send("#{attr}=", (metadata_values + new_values).uniq.join(', ')) - else - new_value = new_value.is_a?(Hash)? new_value.values.first : new_value.to_s - new_value = find_or_create_agent(attr, nil, logger) if enforce?(attr, :Agent) - @submission.send("#{attr}=", new_value) - old_val = @submission.previous_values&.key?(attr) ? @submission.previous_values[attr] : nil - single_extracted = true - end - - unless @submission.valid? - logger.error("Error while extracting metadata for the attribute #{attr}: #{@submission.errors[attr] || @submission.errors}") - new_value&.delete if enforce?(attr, :Agent) && new_value.respond_to?(:delete) - @submission.send("#{attr}=", old_val) - end - - single_extracted - end - - # Return a hash with the best literal value for an URI - # it selects the literal according to their language: no language > english > french > other languages - def select_metadata_literal(metadata_uri, metadata_literal, hash) - return unless metadata_literal.is_a?(RDF::Literal) - - if hash.key?(metadata_uri) - if metadata_literal.has_language? - if !hash[metadata_uri].has_language? - return hash - else - case metadata_literal.language - when :en, :eng - # Take the value with english language over other languages - hash[metadata_uri] = metadata_literal.to_s - return hash - when :fr, :fre - # If no english, take french - if hash[metadata_uri].language == :en || hash[metadata_uri].language == :eng - return hash - else - hash[metadata_uri] = metadata_literal.to_s - return hash - end - else - return hash - end - end - else - # Take the value with no language in priority (considered as a default) - hash[metadata_uri] = metadata_literal.to_s - return hash - end - else - hash[metadata_uri] = metadata_literal.to_s - hash - end - end - - # A function to extract additional metadata - # Take the literal data if the property is pointing to a literal - # If pointing to an URI: first it takes the "omv:name" of the object pointed by the property, if nil it takes the "rdfs:label". - # If not found it check for "omv:firstName + omv:lastName" (for "omv:Person") of this object. And to finish it takes the "URI" - # The hash_results contains the metadataUri (objet pointed on by the metadata property) with the value we are using from it - def extract_each_metadata(ontology_uri, attr, prop_to_extract, logger) - - query_metadata = < #{prop_to_extract} ?extractedObject . - OPTIONAL { ?extractedObject omv:name ?omvname } . - OPTIONAL { ?extractedObject omv:firstName ?omvfirstname } . - OPTIONAL { ?extractedObject omv:lastName ?omvlastname } . - OPTIONAL { ?extractedObject rdfs:label ?rdfslabel } . -} -eos - Goo.namespaces.each do |prefix, uri| - query_metadata = "PREFIX #{prefix}: <#{uri}>\n" + query_metadata - end - - # logger.info(query_metadata) - # This hash will contain the "literal" metadata for each object (uri or literal) pointed by the metadata predicate - hash_results = {} - Goo.sparql_query_client.query(query_metadata).each_solution do |sol| - value = sol[:extractedObject] - if enforce?(attr, :uri) - # If the attr is enforced as URI then it directly takes the URI - uri_value = value ? RDF::URI.new(value.to_s.strip) : nil - hash_results[value] = uri_value if uri_value&.valid? - elsif enforce?(attr, :date_time) - begin - hash_results[value] = DateTime.iso8601(value.to_s) - rescue StandardError => e - logger.error("Impossible to extract DateTime metadata for #{attr}: #{value}. It should follow iso8601 standards. Error message: #{e}") - end - elsif enforce?(attr, :integer) - begin - hash_results[value] = value.to_s.to_i - rescue StandardError => e - logger.error("Impossible to extract integer metadata for #{attr}: #{value}. Error message: #{e}") - end - elsif enforce?(attr, :boolean) - case value.to_s.downcase - when 'true' - hash_results[value] = true - when 'false' - hash_results[value] = false - else - logger.error("Impossible to extract boolean metadata for #{attr}: #{value}. Error message: #{e}") - end - elsif value.is_a?(RDF::URI) - hash_results = find_object_label(hash_results, sol, value) - else - # If this is directly a literal - hash_results = select_metadata_literal(value, value, hash_results) - end - end - hash_results - end - - def find_object_label(hash_results, sol, value) - if !sol[:omvname].nil? - hash_results = select_metadata_literal(value, sol[:omvname], hash_results) - elsif !sol[:rdfslabel].nil? - hash_results = select_metadata_literal(value, sol[:rdfslabel], hash_results) - elsif !sol[:omvfirstname].nil? - hash_results = select_metadata_literal(value, sol[:omvfirstname], hash_results) - # if first and last name are defined (for omv:Person) - hash_results[value] = "#{hash_results[value]} #{sol[:omvlastname]}" unless sol[:omvlastname].nil? - elsif !sol[:omvlastname].nil? - # if only last name is defined - hash_results = select_metadata_literal(value, sol[:omvlastname], hash_results) - else - # if the object is an URI but we are requesting a String - hash_results[value] = value.to_s - end - hash_results - end - - def enforce?(attr, type) - LinkedData::Models::OntologySubmission.attribute_settings(attr)[:enforce].include?(type) - end - - def find_or_create_agent(attr, old_val, logger) - agent = LinkedData::Models::Agent.where(agentType: 'person', name: old_val).first - begin - agent ||= LinkedData::Models::Agent.new(name: old_val, agentType: 'person', creator: @submission.ontology.administeredBy.first).save - rescue - logger.error("Error while extracting metadata for the attribute #{attr}: Can't create Agent #{agent.errors} ") - agent = nil - end - agent - end - end - end - end -end diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/submission_validators.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/submission_validators.rb index 81adc3aa0..93ddb9cb6 100644 --- a/lib/ontologies_linked_data/concerns/ontology_submissions/submission_validators.rb +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/submission_validators.rb @@ -254,7 +254,7 @@ def ontology_has_domain(sub) def default_sparql_endpoint(sub) url = LinkedData.settings.sparql_endpoint_url || '' - url.strip.blank? ? [] : [RDF::URI.new(url)] + url.strip.blank? ? [] : [RDF::URI.new(url)] end def open_search_default(sub) RDF::URI.new("#{LinkedData.settings.rest_url_prefix}search?ontologies=#{sub.ontology.acronym}&q=") diff --git a/lib/ontologies_linked_data/config/config.rb b/lib/ontologies_linked_data/config/config.rb index 0c16c88d8..67b6e43d7 100644 --- a/lib/ontologies_linked_data/config/config.rb +++ b/lib/ontologies_linked_data/config/config.rb @@ -23,8 +23,8 @@ def config(&block) @settings.goo_path_query ||= '/sparql/' @settings.goo_path_data ||= '/data/' @settings.goo_path_update ||= '/update/' - @settings.search_server_url ||= 'http://localhost:8983/solr/term_search_core1' - @settings.property_search_server_url ||= 'http://localhost:8983/solr/prop_search_core1' + @settings.search_server_url ||= 'http://localhost:8983/solr' + @settings.property_search_server_url ||= 'http://localhost:8983/solr' @settings.repository_folder ||= './test/data/ontology_files/repo' @settings.rest_url_prefix ||= DEFAULT_PREFIX @settings.enable_security ||= false @@ -40,8 +40,6 @@ def config(&block) @settings.queries_debug ||= false @settings.enable_monitoring ||= false - @settings.cube_host ||= 'localhost' - @settings.cube_port ||= 1180 # Caching http @settings.enable_http_cache ||= false @@ -87,6 +85,8 @@ def config(&block) # ontology creation to OntoPortal site admins @settings.enable_administrative_notifications ||= true + @settings.oauth_providers ||= {} + # number of times to retry a query when empty records are returned @settings.num_retries_4store ||= 10 @@ -131,14 +131,6 @@ def connect_goo conf.add_search_backend(:property, service: @settings.property_search_server_url) conf.add_redis_backend(host: @settings.goo_redis_host, port: @settings.goo_redis_port) - - if @settings.enable_monitoring - puts "(LD) >> Enable SPARQL monitoring with cube #{@settings.cube_host}:#{@settings.cube_port}" - conf.enable_cube do |opts| - opts[:host] = @settings.cube_host - opts[:port] = @settings.cube_port - end - end end rescue StandardError => e abort("EXITING: Cannot connect to triplestore and/or search server:\n #{e}\n#{e.backtrace.join("\n")}") @@ -171,7 +163,7 @@ def goo_namespaces conf.add_namespace(:adms, RDF::Vocabulary.new("http://www.w3.org/ns/adms#")) conf.add_namespace(:voaf, RDF::Vocabulary.new("http://purl.org/vocommons/voaf#")) conf.add_namespace(:dcat, RDF::Vocabulary.new("http://www.w3.org/ns/dcat#")) - conf.add_namespace(:mod, RDF::Vocabulary.new("http://www.isibang.ac.in/ns/mod#")) + conf.add_namespace(:mod, RDF::Vocabulary.new("https://w3id.org/mod#")) conf.add_namespace(:prov, RDF::Vocabulary.new("http://www.w3.org/ns/prov#")) conf.add_namespace(:cc, RDF::Vocabulary.new("http://creativecommons.org/ns#")) conf.add_namespace(:schema, RDF::Vocabulary.new("http://schema.org/")) diff --git a/lib/ontologies_linked_data/mappings/mappings.rb b/lib/ontologies_linked_data/mappings/mappings.rb index 5d0dedb71..30326b8a9 100644 --- a/lib/ontologies_linked_data/mappings/mappings.rb +++ b/lib/ontologies_linked_data/mappings/mappings.rb @@ -2,704 +2,619 @@ require 'tmpdir' module LinkedData -module Mappings - OUTSTANDING_LIMIT = 30 + module Mappings + OUTSTANDING_LIMIT = 30 + + def self.mapping_predicates + predicates = {} + predicates["CUI"] = ["http://bioportal.bioontology.org/ontologies/umls/cui"] + predicates["SAME_URI"] = + ["http://data.bioontology.org/metadata/def/mappingSameURI"] + predicates["LOOM"] = + ["http://data.bioontology.org/metadata/def/mappingLoom"] + predicates["REST"] = + ["http://data.bioontology.org/metadata/def/mappingRest"] + return predicates + end - def self.mapping_predicates - predicates = {} - predicates["CUI"] = ["http://bioportal.bioontology.org/ontologies/umls/cui"] - predicates["SAME_URI"] = - ["http://data.bioontology.org/metadata/def/mappingSameURI"] - predicates["LOOM"] = - ["http://data.bioontology.org/metadata/def/mappingLoom"] - predicates["REST"] = - ["http://data.bioontology.org/metadata/def/mappingRest"] - return predicates - end + def self.internal_mapping_predicates + predicates = {} + predicates["SKOS:EXACT_MATCH"] = ["http://www.w3.org/2004/02/skos/core#exactMatch"] + predicates["SKOS:CLOSE_MATCH"] = ["http://www.w3.org/2004/02/skos/core#closeMatch"] + predicates["SKOS:BROAD_MATH"] = ["http://www.w3.org/2004/02/skos/core#broadMatch"] + predicates["SKOS:NARROW_MATH"] = ["http://www.w3.org/2004/02/skos/core#narrowMatch"] + predicates["SKOS:RELATED_MATH"] = ["http://www.w3.org/2004/02/skos/core#relatedMatch"] - def self.internal_mapping_predicates - predicates = {} - predicates["SKOS:EXACT_MATCH"] = ["http://www.w3.org/2004/02/skos/core#exactMatch"] - predicates["SKOS:CLOSE_MATCH"] = ["http://www.w3.org/2004/02/skos/core#closeMatch"] - predicates["SKOS:BROAD_MATH"] = ["http://www.w3.org/2004/02/skos/core#broadMatch"] - predicates["SKOS:NARROW_MATH"] = ["http://www.w3.org/2004/02/skos/core#narrowMatch"] - predicates["SKOS:RELATED_MATH"] = ["http://www.w3.org/2004/02/skos/core#relatedMatch"] + return predicates + end - return predicates - end + def self.handle_triple_store_downtime(logger = nil) + epr = Goo.sparql_query_client(:main) + status = epr.status - def self.handle_triple_store_downtime(logger = nil) - epr = Goo.sparql_query_client(:main) - status = epr.status + if status[:exception] + logger.info(status[:exception]) if logger + exit(1) + end - if status[:exception] - logger.info(status[:exception]) if logger - exit(1) + if status[:outstanding] > OUTSTANDING_LIMIT + logger.info("The triple store number of outstanding queries exceeded #{OUTSTANDING_LIMIT}. Exiting...") if logger + exit(1) + end end - if status[:outstanding] > OUTSTANDING_LIMIT - logger.info("The triple store number of outstanding queries exceeded #{OUTSTANDING_LIMIT}. Exiting...") if logger - exit(1) - end - end + def self.mapping_counts(enable_debug=false, logger=nil, reload_cache=false, arr_acronyms=[]) + logger = nil unless enable_debug + t = Time.now + latest = self.retrieve_latest_submissions(options={ acronyms:arr_acronyms }) + counts = {} + i = 0 + epr = Goo.sparql_query_client(:main) - def self.mapping_counts(enable_debug=false, logger=nil, reload_cache=false, arr_acronyms=[]) - logger = nil unless enable_debug - t = Time.now - latest = self.retrieve_latest_submissions(options={ acronyms:arr_acronyms }) - counts = {} - i = 0 - epr = Goo.sparql_query_client(:main) + latest.each do |acro, sub| + self.handle_triple_store_downtime(logger) if Goo.backend_4s? + t0 = Time.now + s_counts = self.mapping_ontologies_count(sub, nil, reload_cache=reload_cache) + s_total = 0 - latest.each do |acro, sub| - self.handle_triple_store_downtime(logger) if LinkedData.settings.goo_backend_name === '4store' - t0 = Time.now - s_counts = self.mapping_ontologies_count(sub, nil, reload_cache=reload_cache) - s_total = 0 + s_counts.each do |k,v| + s_total += v + end + counts[acro] = s_total + i += 1 - s_counts.each do |k,v| - s_total += v + if enable_debug + logger.info("#{i}/#{latest.count} " + + "Retrieved #{s_total} records for #{acro} in #{Time.now - t0} seconds.") + logger.flush + end + sleep(5) end - counts[acro] = s_total - i += 1 if enable_debug - logger.info("#{i}/#{latest.count} " + - "Retrieved #{s_total} records for #{acro} in #{Time.now - t0} seconds.") + logger.info("Total time #{Time.now - t} sec.") logger.flush end - sleep(5) - end - - if enable_debug - logger.info("Total time #{Time.now - t} sec.") - logger.flush - end - return counts - end - - def self.mapping_ontologies_count(sub1, sub2, reload_cache=false) - template = <<-eos -{ - GRAPH <#{sub1.id.to_s}> { - ?s1 ?o . - } - GRAPH graph { - ?s2 ?o . + return counts + end + + def self.mapping_ontologies_count(sub1, sub2, reload_cache=false) + template = <<-eos + { + GRAPH <#{sub1.id.to_s}> { + ?s1 ?o . + } + GRAPH graph { + ?s2 ?o . + } } -} -eos - group_count = sub2.nil? ? {} : nil - count = 0 - latest_sub_ids = self.retrieve_latest_submission_ids - epr = Goo.sparql_query_client(:main) - - mapping_predicates().each do |_source, mapping_predicate| - block = template.gsub("predicate", mapping_predicate[0]) - query_template = <<-eos - SELECT variables - WHERE { - block - filter - } group - eos - query = query_template.sub("block", block) - filter = _source == "SAME_URI" ? '' : 'FILTER (?s1 != ?s2)' + eos + group_count = sub2.nil? ? {} : nil + count = 0 + latest_sub_ids = self.retrieve_latest_submission_ids + epr = Goo.sparql_query_client(:main) - if sub2.nil? - ont_id = sub1.id.to_s.split("/")[0..-3].join("/") - #STRSTARTS is used to not count older graphs - filter += "\nFILTER (!STRSTARTS(str(?g),'#{ont_id}'))" - query = query.sub("graph","?g") - query = query.sub("filter",filter) - query = query.sub("variables","?g (count(?s1) as ?c)") - query = query.sub("group","GROUP BY ?g") - else - query = query.sub("graph","<#{sub2.id.to_s}>") - query = query.sub("filter",filter) - query = query.sub("variables","(count(?s1) as ?c)") - query = query.sub("group","") - end - graphs = [sub1.id, LinkedData::Models::MappingProcess.type_uri] - graphs << sub2.id unless sub2.nil? + mapping_predicates().each do |_source, mapping_predicate| + block = template.gsub("predicate", mapping_predicate[0]) + query_template = <<-eos + SELECT variables + WHERE { + block + filter + } group + eos + query = query_template.sub("block", block) + filter = _source == "SAME_URI" ? '' : 'FILTER (?s1 != ?s2)' + + if sub2.nil? + ont_id = sub1.id.to_s.split("/")[0..-3].join("/") + #STRSTARTS is used to not count older graphs + filter += "\nFILTER (!STRSTARTS(str(?g),'#{ont_id}'))" + query = query.sub("graph","?g") + query = query.sub("filter",filter) + query = query.sub("variables","?g (count(?s1) as ?c)") + query = query.sub("group","GROUP BY ?g") + else + query = query.sub("graph","<#{sub2.id.to_s}>") + query = query.sub("filter",filter) + query = query.sub("variables","(count(?s1) as ?c)") + query = query.sub("group","") + end + graphs = [sub1.id, LinkedData::Models::MappingProcess.type_uri] + graphs << sub2.id unless sub2.nil? - if sub2.nil? - solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) + if sub2.nil? + solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) - solutions.each do |sol| - acr = sol[:g].to_s.split("/")[-3] - next unless latest_sub_ids[acr] == sol[:g].to_s + solutions.each do |sol| + acr = sol[:g].to_s.split("/")[-3] + next unless latest_sub_ids[acr] == sol[:g].to_s - if group_count[acr].nil? - group_count[acr] = 0 + if group_count[acr].nil? + group_count[acr] = 0 + end + group_count[acr] += sol[:c].object + end + else + solutions = epr.query(query, + graphs: graphs ) + solutions.each do |sol| + count += sol[:c].object end - group_count[acr] += sol[:c].object - end - else - solutions = epr.query(query, - graphs: graphs ) - solutions.each do |sol| - count += sol[:c].object end + end #per predicate query + + if sub2.nil? + return group_count end - end #per predicate query + return count + end - if sub2.nil? - return group_count + def self.empty_page(page,size) + p = Goo::Base::Page.new(page,size,nil,[]) + p.aggregate = 0 + return p end - return count - end - def self.empty_page(page,size) - p = Goo::Base::Page.new(page,size,nil,[]) - p.aggregate = 0 - return p - end + def self.mappings_ontologies(sub1, sub2, page, size, classId = nil, reload_cache = false) + sub1, acr1 = extract_acronym(sub1) + sub2, acr2 = extract_acronym(sub2) - def self.mappings_ontologies(sub1, sub2, page, size, classId = nil, reload_cache = false) - sub1, acr1 = extract_acronym(sub1) - sub2, acr2 = extract_acronym(sub2) + mappings = [] + persistent_count = 0 - mappings = [] - persistent_count = 0 + if classId.nil? + persistent_count = count_mappings(acr1, acr2) + return LinkedData::Mappings.empty_page(page, size) if persistent_count == 0 + end - if classId.nil? - persistent_count = count_mappings(acr1, acr2) - return LinkedData::Mappings.empty_page(page, size) if persistent_count == 0 - end + query = mappings_ont_build_query(classId, page, size, sub1, sub2) + epr = Goo.sparql_query_client(:main) + graphs = [sub1] + unless sub2.nil? + graphs << sub2 + end + solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) + s1 = nil + s1 = RDF::URI.new(classId.to_s) unless classId.nil? - query = mappings_ont_build_query(classId, page, size, sub1, sub2) - epr = Goo.sparql_query_client(:main) - graphs = [sub1] - unless sub2.nil? - graphs << sub2 - end - solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) - s1 = nil - s1 = RDF::URI.new(classId.to_s) unless classId.nil? + solutions.each do |sol| + graph2 = sub2.nil? ? sol[:g] : sub2 + s1 = sol[:s1] if classId.nil? + backup_mapping = nil + + if sol[:source].to_s == "REST" + backup_mapping = LinkedData::Models::RestBackupMapping + .find(sol[:o]).include(:process, :class_urns).first + backup_mapping.process.bring_remaining + end - solutions.each do |sol| - graph2 = sub2.nil? ? sol[:g] : sub2 - s1 = sol[:s1] if classId.nil? - backup_mapping = nil - - if sol[:source].to_s == "REST" - backup_mapping = LinkedData::Models::RestBackupMapping - .find(sol[:o]).include(:process, :class_urns).first - backup_mapping.process.bring_remaining - end + classes = get_mapping_classes_instance(s1, sub1, sol[:s2], graph2) - classes = get_mapping_classes_instance(s1, sub1, sol[:s2], graph2) + mapping = if backup_mapping.nil? + LinkedData::Models::Mapping.new(classes, sol[:source].to_s) + else + LinkedData::Models::Mapping.new( + classes, sol[:source].to_s, + backup_mapping.process, backup_mapping.id) + end - mapping = if backup_mapping.nil? - LinkedData::Models::Mapping.new(classes, sol[:source].to_s) - else - LinkedData::Models::Mapping.new( - classes, sol[:source].to_s, - backup_mapping.process, backup_mapping.id) - end + mappings << mapping + end - mappings << mapping - end + if size == 0 + return mappings + end - if size == 0 - return mappings + page = Goo::Base::Page.new(page, size, persistent_count, mappings) + return page end - page = Goo::Base::Page.new(page, size, persistent_count, mappings) - return page + def self.mappings_ontology(sub,page,size,classId=nil,reload_cache=false) + return self.mappings_ontologies(sub,nil,page,size,classId=classId, + reload_cache=reload_cache) end - def self.mappings_ontology(sub,page,size,classId=nil,reload_cache=false) - return self.mappings_ontologies(sub,nil,page,size,classId=classId, - reload_cache=reload_cache) - end - - def self.read_only_class(classId,submissionId) - ontologyId = submissionId - acronym = nil - unless submissionId["submissions"].nil? - ontologyId = submissionId.split("/")[0..-3] - acronym = ontologyId.last - ontologyId = ontologyId.join("/") - else - acronym = ontologyId.split("/")[-1] - end - ontology = LinkedData::Models::Ontology - .read_only( - id: RDF::IRI.new(ontologyId), - acronym: acronym) - submission = LinkedData::Models::OntologySubmission - .read_only( - id: RDF::IRI.new(ontologyId+"/submissions/latest"), - # id: RDF::IRI.new(submissionId), - ontology: ontology) - mappedClass = LinkedData::Models::Class - .read_only( - id: RDF::IRI.new(classId), - submission: submission, - urn_id: LinkedData::Models::Class.urn_id(acronym,classId) ) - return mappedClass - end - - def self.migrate_rest_mappings(acronym) - mappings = LinkedData::Models::RestBackupMapping - .where.include(:uuid, :class_urns, :process).all - if mappings.length == 0 - return [] - end - triples = [] - - rest_predicate = mapping_predicates()["REST"][0] - mappings.each do |m| - m.class_urns.each do |u| - u = u.to_s - if u.start_with?("urn:#{acronym}") - class_id = u.split(":")[2..-1].join(":") - triples << - " <#{class_id}> <#{rest_predicate}> <#{m.id}> . " + def self.read_only_class(classId,submissionId) + ontologyId = submissionId + acronym = nil + unless submissionId["submissions"].nil? + ontologyId = submissionId.split("/")[0..-3] + acronym = ontologyId.last + ontologyId = ontologyId.join("/") + else + acronym = ontologyId.split("/")[-1] end + ontology = LinkedData::Models::Ontology + .read_only( + id: RDF::IRI.new(ontologyId), + acronym: acronym) + submission = LinkedData::Models::OntologySubmission + .read_only( + id: RDF::IRI.new(ontologyId+"/submissions/latest"), + # id: RDF::IRI.new(submissionId), + ontology: ontology) + mappedClass = LinkedData::Models::Class + .read_only( + id: RDF::IRI.new(classId), + submission: submission, + urn_id: LinkedData::Models::Class.urn_id(acronym,classId) ) + return mappedClass + end + + def self.migrate_rest_mappings(acronym) + mappings = LinkedData::Models::RestBackupMapping + .where.include(:uuid, :class_urns, :process).all + if mappings.length == 0 + return [] end + triples = [] + + rest_predicate = mapping_predicates()["REST"][0] + mappings.each do |m| + m.class_urns.each do |u| + u = u.to_s + if u.start_with?("urn:#{acronym}") + class_id = u.split(":")[2..-1].join(":") + triples << + " <#{class_id}> <#{rest_predicate}> <#{m.id}> . " + end + end + end + return triples end - return triples - end - def self.delete_rest_mapping(mapping_id) - mapping = get_rest_mapping(mapping_id) - if mapping.nil? - return nil - end - rest_predicate = mapping_predicates()["REST"][0] - classes = mapping.classes - classes.each do |c| - sub = c.submission - unless sub.id.to_s["latest"].nil? - #the submission in the class might point to latest - sub = LinkedData::Models::Ontology.find(c.submission.ontology.id) - .first - .latest_submission - end - graph_delete = RDF::Graph.new - graph_delete << [c.id, RDF::URI.new(rest_predicate), mapping.id] - Goo.sparql_update_client.delete_data(graph_delete, graph: sub.id) + def self.delete_rest_mapping(mapping_id) + mapping = get_rest_mapping(mapping_id) + if mapping.nil? + return nil + end + rest_predicate = mapping_predicates()["REST"][0] + classes = mapping.classes + classes.each do |c| + sub = c.submission + unless sub.id.to_s["latest"].nil? + #the submission in the class might point to latest + sub = LinkedData::Models::Ontology.find(c.submission.ontology.id) + .first + .latest_submission + end + graph_delete = RDF::Graph.new + graph_delete << [RDF::URI.new(c.id), RDF::URI.new(rest_predicate), mapping.id] + Goo.sparql_update_client.delete_data(graph_delete, graph: sub.id) + end + mapping.process.delete + backup = LinkedData::Models::RestBackupMapping.find(mapping_id).first + unless backup.nil? + backup.delete + end + return mapping end - mapping.process.delete - backup = LinkedData::Models::RestBackupMapping.find(mapping_id).first - unless backup.nil? - backup.delete + + def self.get_rest_mapping(mapping_id) + backup = LinkedData::Models::RestBackupMapping.find(mapping_id).first + if backup.nil? + return nil + end + rest_predicate = mapping_predicates()["REST"][0] + qmappings = <<-eos + SELECT DISTINCT ?s1 ?c1 ?s2 ?c2 ?uuid ?o + WHERE { + ?uuid ?o . + + GRAPH ?s1 { + ?c1 <#{rest_predicate}> ?uuid . + } + GRAPH ?s2 { + ?c2 <#{rest_predicate}> ?uuid . + } + FILTER(?uuid = <#{LinkedData::Models::Base.replace_url_prefix_to_id(mapping_id)}>) + FILTER(?s1 != ?s2) + } LIMIT 1 + eos + epr = Goo.sparql_query_client(:main) + graphs = [LinkedData::Models::MappingProcess.type_uri] + mapping = nil + epr.query(qmappings, + graphs: graphs).each do |sol| + classes = [read_only_class(sol[:c1].to_s, sol[:s1].to_s), + read_only_class(sol[:c2].to_s, sol[:s2].to_s)] + process = LinkedData::Models::MappingProcess.find(sol[:o]).first + mapping = LinkedData::Models::Mapping.new(classes, 'REST', + process, + sol[:uuid]) + end + return mapping end - return mapping - end - def self.get_rest_mapping(mapping_id) - backup = LinkedData::Models::RestBackupMapping.find(mapping_id).first - if backup.nil? - return nil + def self.create_rest_mapping(classes,process) + unless process.instance_of? LinkedData::Models::MappingProcess + raise ArgumentError, "Process should be instance of MappingProcess" + end + if classes.length != 2 + raise ArgumentError, "Create REST is avalaible for two classes. " + + "Request contains #{classes.length} classes." + end + #first create back up mapping that lives across submissions + backup_mapping = LinkedData::Models::RestBackupMapping.new + backup_mapping.uuid = UUID.new.generate + backup_mapping.process = process + class_urns = [] + classes.each do |c| + if c.instance_of?LinkedData::Models::Class + acronym = c.submission.id.to_s.split("/")[-3] + class_urns << RDF::URI.new( + LinkedData::Models::Class.urn_id(acronym,c.id.to_s)) + + else + class_urns << RDF::URI.new(c.urn_id()) + end + end + backup_mapping.class_urns = class_urns + backup_mapping.save + + #second add the mapping id to current submission graphs + rest_predicate = mapping_predicates()["REST"][0] + classes.each do |c| + sub = c.submission + unless sub.id.to_s["latest"].nil? + #the submission in the class might point to latest + sub = LinkedData::Models::Ontology.find(c.submission.ontology.id).first.latest_submission + end + graph_insert = RDF::Graph.new + graph_insert << [c.id, RDF::URI.new(rest_predicate), backup_mapping.id] + Goo.sparql_update_client.insert_data(graph_insert, graph: sub.id) + end + mapping = LinkedData::Models::Mapping.new(classes,"REST", process, backup_mapping.id) + return mapping end - rest_predicate = mapping_predicates()["REST"][0] - qmappings = <<-eos -SELECT DISTINCT ?s1 ?c1 ?s2 ?c2 ?uuid ?o -WHERE { - ?uuid ?o . - GRAPH ?s1 { - ?c1 <#{rest_predicate}> ?uuid . - } - GRAPH ?s2 { - ?c2 <#{rest_predicate}> ?uuid . + def self.mappings_for_classids(class_ids,sources=["REST","CUI"]) + class_ids = class_ids.uniq + predicates = {} + sources.each do |t| + predicates[mapping_predicates()[t][0]] = t + end + qmappings = <<-eos + SELECT DISTINCT ?s1 ?c1 ?s2 ?c2 ?pred + WHERE { + GRAPH ?s1 { + ?c1 ?pred ?o . + } + GRAPH ?s2 { + ?c2 ?pred ?o . + } + FILTER(?s1 != ?s2) + FILTER(filter_pred) + FILTER(filter_classes) } -FILTER(?uuid = <#{LinkedData::Models::Base.replace_url_prefix_to_id(mapping_id)}>) -FILTER(?s1 != ?s2) -} LIMIT 1 - eos + eos + qmappings = qmappings.gsub("filter_pred", + predicates.keys.map { |x| "?pred = <#{x}>"}.join(" || ")) + qmappings = qmappings.gsub("filter_classes", + class_ids.map { |x| "?c1 = <#{x}>" }.join(" || ")) epr = Goo.sparql_query_client(:main) graphs = [LinkedData::Models::MappingProcess.type_uri] - mapping = nil + mappings = [] epr.query(qmappings, graphs: graphs).each do |sol| - classes = [read_only_class(sol[:c1].to_s, sol[:s1].to_s), - read_only_class(sol[:c2].to_s, sol[:s2].to_s)] - process = LinkedData::Models::MappingProcess.find(sol[:o]).first - mapping = LinkedData::Models::Mapping.new(classes, 'REST', - process, - sol[:uuid]) - end - return mapping - end - - def self.create_rest_mapping(classes,process) - unless process.instance_of? LinkedData::Models::MappingProcess - raise ArgumentError, "Process should be instance of MappingProcess" - end - if classes.length != 2 - raise ArgumentError, "Create REST is avalaible for two classes. " + - "Request contains #{classes.length} classes." - end - #first create back up mapping that lives across submissions - backup_mapping = LinkedData::Models::RestBackupMapping.new - backup_mapping.uuid = UUID.new.generate - backup_mapping.process = process - class_urns = [] - classes.each do |c| - if c.instance_of?LinkedData::Models::Class - acronym = c.submission.id.to_s.split("/")[-3] - class_urns << RDF::URI.new( - LinkedData::Models::Class.urn_id(acronym,c.id.to_s)) - - else - class_urns << RDF::URI.new(c.urn_id()) + classes = [ read_only_class(sol[:c1].to_s,sol[:s1].to_s), + read_only_class(sol[:c2].to_s,sol[:s2].to_s) ] + source = predicates[sol[:pred].to_s] + mappings << LinkedData::Models::Mapping.new(classes,source) end + return mappings end - backup_mapping.class_urns = class_urns - backup_mapping.save - - #second add the mapping id to current submission graphs - rest_predicate = mapping_predicates()["REST"][0] - classes.each do |c| - sub = c.submission - unless sub.id.to_s["latest"].nil? - #the submission in the class might point to latest - sub = LinkedData::Models::Ontology.find(c.submission.ontology.id).first.latest_submission - end - graph_insert = RDF::Graph.new - graph_insert << [c.id, RDF::URI.new(rest_predicate), backup_mapping.id] - Goo.sparql_update_client.insert_data(graph_insert, graph: sub.id) - end - mapping = LinkedData::Models::Mapping.new(classes,"REST", process, backup_mapping.id) - return mapping - end - def self.mappings_for_classids(class_ids,sources=["REST","CUI"]) - class_ids = class_ids.uniq - predicates = {} - sources.each do |t| - predicates[mapping_predicates()[t][0]] = t - end - qmappings = <<-eos -SELECT DISTINCT ?s1 ?c1 ?s2 ?c2 ?pred -WHERE { - GRAPH ?s1 { - ?c1 ?pred ?o . + def self.recent_rest_mappings(n) + graphs = [LinkedData::Models::MappingProcess.type_uri] + qdate = <<-eos + SELECT DISTINCT ?s + FROM <#{LinkedData::Models::MappingProcess.type_uri}> + WHERE { ?s ?o } + ORDER BY DESC(?o) LIMIT #{n} + eos + epr = Goo.sparql_query_client(:main) + procs = [] + epr.query(qdate, graphs: graphs,query_options: {rules: :NONE}).each do |sol| + procs << sol[:s] + end + if procs.length == 0 + return [] + end + graphs = [LinkedData::Models::MappingProcess.type_uri] + proc_object = Hash.new + LinkedData::Models::MappingProcess.where + .include(LinkedData::Models::MappingProcess.attributes) + .all.each do |obj| + #highly cached query + proc_object[obj.id.to_s] = obj + end + procs = procs.map { |x| "?o = #{x.to_ntriples}" }.join " || " + rest_predicate = mapping_predicates()["REST"][0] + qmappings = <<-eos + SELECT DISTINCT ?ont1 ?c1 ?ont2 ?c2 ?o ?uuid + WHERE { + ?uuid ?o . + + ?s1 ?ont1 . + GRAPH ?s1 { + ?c1 <#{rest_predicate}> ?uuid . + } + ?s2 ?ont2 . + GRAPH ?s2 { + ?c2 <#{rest_predicate}> ?uuid . + } + FILTER(?ont1 != ?ont2) + FILTER(?c1 != ?c2) + FILTER (#{procs}) } - GRAPH ?s2 { - ?c2 ?pred ?o . + eos + epr = Goo.sparql_query_client(:main) + mappings = [] + epr.query(qmappings, + graphs: graphs,query_options: {rules: :NONE}).each do |sol| + classes = [ read_only_class(sol[:c1].to_s,sol[:ont1].to_s), + read_only_class(sol[:c2].to_s,sol[:ont2].to_s) ] + process = proc_object[sol[:o].to_s] + mapping = LinkedData::Models::Mapping.new(classes,"REST", + process, + sol[:uuid]) + mappings << mapping + end + return mappings.sort_by { |x| x.process.date }.reverse[0..n-1] + end + + def self.retrieve_latest_submission_ids(options = {}) + include_views = options[:include_views] || false + ids_query = <<-eos + PREFIX xsd: + SELECT (CONCAT(xsd:string(?ontology), "/submissions/", xsd:string(MAX(?submissionId))) as ?id) + WHERE { + ?id ?ontology . + ?id ?submissionId . + ?id ?submissionStatus . + ?submissionStatus "RDF" . + include_views_filter } -FILTER(?s1 != ?s2) -FILTER(filter_pred) -FILTER(filter_classes) -} -eos - qmappings = qmappings.gsub("filter_pred", - predicates.keys.map { |x| "?pred = <#{x}>"}.join(" || ")) - qmappings = qmappings.gsub("filter_classes", - class_ids.map { |x| "?c1 = <#{x}>" }.join(" || ")) - epr = Goo.sparql_query_client(:main) - graphs = [LinkedData::Models::MappingProcess.type_uri] - mappings = [] - epr.query(qmappings, - graphs: graphs).each do |sol| - classes = [ read_only_class(sol[:c1].to_s,sol[:s1].to_s), - read_only_class(sol[:c2].to_s,sol[:s2].to_s) ] - source = predicates[sol[:pred].to_s] - mappings << LinkedData::Models::Mapping.new(classes,source) - end - return mappings - end - - def self.recent_rest_mappings(n) - graphs = [LinkedData::Models::MappingProcess.type_uri] - qdate = <<-eos -SELECT DISTINCT ?s -FROM <#{LinkedData::Models::MappingProcess.type_uri}> -WHERE { ?s ?o } -ORDER BY DESC(?o) LIMIT #{n} -eos - epr = Goo.sparql_query_client(:main) - procs = [] - epr.query(qdate, graphs: graphs,query_options: {rules: :NONE}).each do |sol| - procs << sol[:s] - end - if procs.length == 0 - return [] - end - graphs = [LinkedData::Models::MappingProcess.type_uri] - proc_object = Hash.new - LinkedData::Models::MappingProcess.where - .include(LinkedData::Models::MappingProcess.attributes) - .all.each do |obj| - #highly cached query - proc_object[obj.id.to_s] = obj - end - procs = procs.map { |x| "?o = #{x.to_ntriples}" }.join " || " - rest_predicate = mapping_predicates()["REST"][0] - qmappings = <<-eos -SELECT DISTINCT ?ont1 ?c1 ?ont2 ?c2 ?o ?uuid -WHERE { - ?uuid ?o . + GROUP BY ?ontology + eos + include_views_filter = include_views ? '' : <<-eos + OPTIONAL { + ?id ?ontJoin . + } + OPTIONAL { + ?ontJoin ?viewOf . + } + FILTER(!BOUND(?viewOf)) + eos + ids_query.gsub!("include_views_filter", include_views_filter) + epr = Goo.sparql_query_client(:main) + solutions = epr.query(ids_query) + latest_ids = {} - ?s1 ?ont1 . - GRAPH ?s1 { - ?c1 <#{rest_predicate}> ?uuid . - } - ?s2 ?ont2 . - GRAPH ?s2 { - ?c2 <#{rest_predicate}> ?uuid . - } -FILTER(?ont1 != ?ont2) -FILTER(?c1 != ?c2) -FILTER (#{procs}) -} -eos - epr = Goo.sparql_query_client(:main) - mappings = [] - epr.query(qmappings, - graphs: graphs,query_options: {rules: :NONE}).each do |sol| - classes = [ read_only_class(sol[:c1].to_s,sol[:ont1].to_s), - read_only_class(sol[:c2].to_s,sol[:ont2].to_s) ] - process = proc_object[sol[:o].to_s] - mapping = LinkedData::Models::Mapping.new(classes,"REST", - process, - sol[:uuid]) - mappings << mapping - end - return mappings.sort_by { |x| x.process.date }.reverse[0..n-1] - end + solutions.each do |sol| + acr = sol[:id].to_s.split("/")[-3] + latest_ids[acr] = sol[:id].object + end - def self.retrieve_latest_submission_ids(options = {}) - include_views = options[:include_views] || false - ids_query = <<-eos -PREFIX xsd: -SELECT (CONCAT(xsd:string(?ontology), "/submissions/", xsd:string(MAX(?submissionId))) as ?id) -WHERE { - ?id ?ontology . - ?id ?submissionId . - ?id ?submissionStatus . - ?submissionStatus "RDF" . - include_views_filter -} -GROUP BY ?ontology - eos - include_views_filter = include_views ? '' : <<-eos - OPTIONAL { - ?id ?ontJoin . - } - OPTIONAL { - ?ontJoin ?viewOf . - } - FILTER(!BOUND(?viewOf)) - eos - ids_query.gsub!("include_views_filter", include_views_filter) - epr = Goo.sparql_query_client(:main) - solutions = epr.query(ids_query) - latest_ids = {} - - solutions.each do |sol| - acr = sol[:id].to_s.split("/")[-3] - latest_ids[acr] = sol[:id].object + latest_ids end - latest_ids - end + def self.retrieve_latest_submissions(options = {}) + acronyms = (options[:acronyms] || []) + status = (options[:status] || "RDF").to_s.upcase + include_ready = status.eql?("READY") ? true : false + status = "RDF" if status.eql?("READY") + any = status.eql?("ANY") + include_views = options[:include_views] || false - def self.retrieve_latest_submissions(options = {}) - acronyms = (options[:acronyms] || []) - status = (options[:status] || "RDF").to_s.upcase - include_ready = status.eql?("READY") ? true : false - status = "RDF" if status.eql?("READY") - any = status.eql?("ANY") - include_views = options[:include_views] || false - - if any - submissions_query = LinkedData::Models::OntologySubmission.where - else - submissions_query = LinkedData::Models::OntologySubmission.where(submissionStatus: [code: status]) - end - submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views - submissions = submissions_query.include(:submissionStatus,:submissionId, ontology: [:acronym]).to_a - submissions.select! { |sub| acronyms.include?(sub.ontology.acronym) } unless acronyms.empty? - latest_submissions = {} - - submissions.each do |sub| - next if include_ready && !sub.ready? - latest_submissions[sub.ontology.acronym] ||= sub - latest_submissions[sub.ontology.acronym] = sub if sub.submissionId > latest_submissions[sub.ontology.acronym].submissionId + if any + submissions_query = LinkedData::Models::OntologySubmission.where + else + submissions_query = LinkedData::Models::OntologySubmission.where(submissionStatus: [code: status]) + end + submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions = submissions_query.include(:submissionStatus,:submissionId, ontology: [:acronym]).to_a + submissions.select! { |sub| acronyms.include?(sub.ontology.acronym) } unless acronyms.empty? + latest_submissions = {} + + submissions.each do |sub| + next if include_ready && !sub.ready? + latest_submissions[sub.ontology.acronym] ||= sub + latest_submissions[sub.ontology.acronym] = sub if sub.submissionId > latest_submissions[sub.ontology.acronym].submissionId + end + return latest_submissions end - return latest_submissions - end - def self.create_mapping_counts(logger, arr_acronyms=[]) - ont_msg = arr_acronyms.empty? ? "all ontologies" : "ontologies [#{arr_acronyms.join(', ')}]" + def self.create_mapping_counts(logger, arr_acronyms = []) + ont_msg = arr_acronyms.empty? ? 'all ontologies' : "ontologies [#{arr_acronyms.join(', ')}]" - time = Benchmark.realtime do - self.create_mapping_count_totals_for_ontologies(logger, arr_acronyms) - end - logger.info("Completed rebuilding total mapping counts for #{ont_msg} in #{(time/60).round(1)} minutes.") - - time = Benchmark.realtime do - self.create_mapping_count_pairs_for_ontologies(logger, arr_acronyms) - end - logger.info("Completed rebuilding mapping count pairs for #{ont_msg} in #{(time/60).round(1)} minutes.") - end + time = Benchmark.realtime do + self.create_mapping_count_totals_for_ontologies(logger, arr_acronyms) + end + logger.info("Completed rebuilding total mapping counts for #{ont_msg} in #{(time / 60).round(1)} minutes.") + puts "create mappings total count time: #{time}" - def self.create_mapping_count_totals_for_ontologies(logger, arr_acronyms) - new_counts = self.mapping_counts(enable_debug=true, logger=logger, reload_cache=true, arr_acronyms) - persistent_counts = {} - f = Goo::Filter.new(:pair_count) == false - LinkedData::Models::MappingCount.where.filter(f) - .include(:ontologies, :count) - .include(:all) - .all - .each do |m| - persistent_counts[m.ontologies.first] = m + time = Benchmark.realtime do + self.create_mapping_count_pairs_for_ontologies(logger, arr_acronyms) + end + puts "create mappings pair count time: #{time}" + logger.info("Completed rebuilding mapping count pairs for #{ont_msg} in #{(time / 60).round(1)} minutes.") end - num_counts = new_counts.keys.length - ctr = 0 + def self.create_mapping_count_totals_for_ontologies(logger, arr_acronyms) + new_counts = mapping_counts(true, logger, true, arr_acronyms) + persistent_counts = self.all_existent_mapping_counts(pair_count: false) + latest = retrieve_latest_submissions + self.delete_zombie_submission_count(persistent_counts, latest) - new_counts.each_key do |acr| - new_count = new_counts[acr] - ctr += 1 + num_counts = new_counts.keys.length + ctr = 0 - if persistent_counts.include?(acr) - inst = persistent_counts[acr] + new_counts.each_key do |acr| + new_count = new_counts[acr] + ctr += 1 + self.update_mapping_count(persistent_counts, new_counts, acr, acr, new_count, false) + remaining = num_counts - ctr + logger.info("Total mapping count saved for #{acr}: #{new_count}. " << (remaining.positive? ? "#{remaining} counts remaining..." : 'All done!')) + end + end - if new_count != inst.count - inst.bring_remaining - inst.count = new_count + # This generates pair mapping counts for the given + # ontologies to ALL other ontologies in the system + def self.create_mapping_count_pairs_for_ontologies(logger, arr_acronyms) + all_latest_submissions = retrieve_latest_submissions + latest_submissions = if Array(arr_acronyms).empty? + all_latest_submissions + else + all_latest_submissions.select { |k, _v| arr_acronyms.include?(k) } + end - begin - if inst.valid? - inst.save - else - logger.error("Error updating mapping count for #{acr}: #{inst.id.to_s}. #{inst.errors}") - next - end - rescue Exception => e - logger.error("Exception updating mapping count for #{acr}: #{inst.id.to_s}. #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}") - next - end - end - else - m = LinkedData::Models::MappingCount.new - m.ontologies = [acr] - m.pair_count = false - m.count = new_count + ont_total = latest_submissions.length + logger.info("There is a total of #{ont_total} ontologies to process...") + ont_ctr = 0 - begin - if m.valid? - m.save - else - logger.error("Error saving new mapping count for #{acr}. #{m.errors}") - next - end - rescue Exception => e - logger.error("Exception saving new mapping count for #{acr}. #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}") - next - end - end - remaining = num_counts - ctr - logger.info("Total mapping count saved for #{acr}: #{new_count}. " << ((remaining > 0) ? "#{remaining} counts remaining..." : "All done!")) - end - end + persistent_counts = self.all_existent_mapping_counts(pair_count: true) + self.delete_zombie_submission_count(persistent_counts, all_latest_submissions) - # This generates pair mapping counts for the given - # ontologies to ALL other ontologies in the system - def self.create_mapping_count_pairs_for_ontologies(logger, arr_acronyms) - latest_submissions = self.retrieve_latest_submissions(options={acronyms:arr_acronyms}) - ont_total = latest_submissions.length - logger.info("There is a total of #{ont_total} ontologies to process...") - ont_ctr = 0 - # filename = 'mapping_pairs.ttl' - # temp_dir = Dir.tmpdir - # temp_file_path = File.join(temp_dir, filename) - # temp_dir = '/Users/mdorf/Downloads/test/' - # temp_file_path = File.join(File.dirname(file_path), "test.ttl") - # fsave = File.open(temp_file_path, "a") - - latest_submissions.each do |acr, sub| - self.handle_triple_store_downtime(logger) if LinkedData.settings.goo_backend_name === '4store' - new_counts = nil - time = Benchmark.realtime do - new_counts = self.mapping_ontologies_count(sub, nil, reload_cache=true) - end - logger.info("Retrieved new mapping pair counts for #{acr} in #{time} seconds.") - ont_ctr += 1 - persistent_counts = {} - LinkedData::Models::MappingCount.where(pair_count: true).and(ontologies: acr) - .include(:ontologies, :count).all.each do |m| - other = m.ontologies.first + latest_submissions.each do |acr, sub| + self.handle_triple_store_downtime(logger) if Goo.backend_4s? + new_counts = nil - if other == acr - other = m.ontologies[1] + time = Benchmark.realtime do + new_counts = mapping_ontologies_count(sub, nil, true) end - persistent_counts[other] = m - end + logger.info("Retrieved new mapping pair counts for #{acr} in #{time} seconds.") + ont_ctr += 1 - num_counts = new_counts.keys.length - logger.info("Ontology: #{acr}. #{num_counts} mapping pair counts to record...") - logger.info("------------------------------------------------") - ctr = 0 + persistent_counts = self.all_existent_mapping_counts(acr: acr, pair_count: true) + self.delete_zombie_mapping_count(persistent_counts, new_counts) - new_counts.each_key do |other| - new_count = new_counts[other] - ctr += 1 + num_counts = new_counts.keys.length + logger.info("Ontology: #{acr}. #{num_counts} mapping pair counts to record...") + logger.info('------------------------------------------------') + ctr = 0 - if persistent_counts.include?(other) - inst = persistent_counts[other] + new_counts.each_key do |other| + new_count = new_counts[other] + ctr += 1 + self.update_mapping_count(persistent_counts, new_counts, acr, other, new_count, true) + remaining = num_counts - ctr + logger.info("Mapping count saved for the pair [#{acr}, #{other}]: #{new_count}. " << (remaining.positive? ? "#{remaining} counts remaining for #{acr}..." : 'All done!')) + wait_interval = 250 - if new_count != inst.count - inst.bring_remaining - inst.pair_count = true - inst.count = new_count - - begin - if inst.valid? - inst.save() - # inst.save({ batch: fsave }) - else - logger.error("Error updating mapping count for the pair [#{acr}, #{other}]: #{inst.id.to_s}. #{inst.errors}") - next - end - rescue Exception => e - logger.error("Exception updating mapping count for the pair [#{acr}, #{other}]: #{inst.id.to_s}. #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}") - next - end - end - else - m = LinkedData::Models::MappingCount.new - m.count = new_count - m.ontologies = [acr,other] - m.pair_count = true - - begin - if m.valid? - m.save() - # m.save({ batch: fsave }) - else - logger.error("Error saving new mapping count for the pair [#{acr}, #{other}]. #{m.errors}") - next - end - rescue Exception => e - logger.error("Exception saving new mapping count for the pair [#{acr}, #{other}]. #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}") - next - end - end - remaining = num_counts - ctr - logger.info("Mapping count saved for the pair [#{acr}, #{other}]: #{new_count}. " << ((remaining > 0) ? "#{remaining} counts remaining for #{acr}..." : "All done!")) - wait_interval = 250 + next unless (ctr % wait_interval).zero? - if ctr % wait_interval == 0 sec_to_wait = 1 logger.info("Waiting #{sec_to_wait} second" << ((sec_to_wait > 1) ? 's' : '') << '...') sleep(sec_to_wait) end + remaining_ont = ont_total - ont_ctr + logger.info("Completed processing pair mapping counts for #{acr}. " << (remaining_ont.positive? ? "#{remaining_ont} ontologies remaining..." : 'All ontologies processed!')) end - remaining_ont = ont_total - ont_ctr - logger.info("Completed processing pair mapping counts for #{acr}. " << ((remaining_ont > 0) ? "#{remaining_ont} ontologies remaining..." : "All ontologies processed!")) - sleep(5) + # fsave.close end - # fsave.close - end private @@ -716,11 +631,6 @@ def self.mappings_ont_build_query(class_id, page, size, sub1, sub2) "BIND ('#{_source}' AS ?source)") end - - - - - filter = class_id.nil? ? "FILTER ((?s1 != ?s2) || (?source = 'SAME_URI'))" : '' if sub2.nil? @@ -772,7 +682,7 @@ def self.mappings_ont_build_query(class_id, page, size, sub1, sub2) def self.mappings_union_template(class_id, sub1, sub2, predicate, bind) class_id_subject = class_id.nil? ? '?s1' : "<#{class_id.to_s}>" target_graph = sub2.nil? ? '?g' : "<#{sub2.to_s}>" - union_template = <<-eos + return <<-eos { GRAPH <#{sub1.to_s}> { #{class_id_subject} <#{predicate}> ?o . @@ -810,6 +720,83 @@ def self.extract_acronym(submission) return sub, acr end + def self.calculate_and_log_counts(uri, logger, reload_cache, label, enable_debug) + start_time = Time.now + count = mapping_ontologies_count(uri, nil, reload_cache).values.sum + logger&.info("#{label} took #{Time.now - start_time} sec. records #{count}") if enable_debug + count + end + + def self.update_mapping_count(persistent_counts, new_counts, acr, other, new_count, pair_count) + if persistent_counts.key?(other) + inst = persistent_counts[other] + if new_count.zero? && pair_count + inst.delete + elsif new_count != inst.count + inst.pair_count = pair_count + inst.count = new_count + inst.save + end + else + return if !new_counts.key?(other) && pair_count + + m = LinkedData::Models::MappingCount.new + m.count = new_count + m.ontologies = if pair_count + [acr, other] + else + [acr] + end + m.pair_count = pair_count + return if m.exist? + + m.save + end + end + + def self.delete_zombie_mapping_count(persistent_counts, new_counts) + persistent_counts.each do |acronym, mapping| + next if mapping.ontologies.all? { |x| new_counts.key?(x) } + + mapping.delete + persistent_counts.delete(acronym) + end + end + + def self.delete_zombie_submission_count(persistent_counts, all_latest_submissions) + special_mappings = ["http://data.bioontology.org/metadata/ExternalMappings", + "http://data.bioontology.org/metadata/InterportalMappings/agroportal", + "http://data.bioontology.org/metadata/InterportalMappings/ncbo", + "http://data.bioontology.org/metadata/InterportalMappings/sifr"] + + persistent_counts.each do |acronym, mapping| + next if mapping.ontologies.size == 1 && !(mapping.ontologies & special_mappings).empty? + next if mapping.ontologies.all? { |x| all_latest_submissions.key?(x) } + + mapping.delete + persistent_counts.delete(acronym) + end + end + + def self.all_existent_mapping_counts(acr: nil, pair_count: true) + persistent_counts = {} + query = LinkedData::Models::MappingCount + query = if acr + query.where(ontologies: acr) + else + query.where + end + + f = Goo::Filter.new(:pair_count) == pair_count + query = query.filter(f) + + query.include(:ontologies, :count).all.each do |m| + other = m.ontologies.first + other = m.ontologies.last if acr && (other == acr) + persistent_counts[other] = m + end + persistent_counts + end end end diff --git a/lib/ontologies_linked_data/media_types.rb b/lib/ontologies_linked_data/media_types.rb index d109e80db..ec8257316 100644 --- a/lib/ontologies_linked_data/media_types.rb +++ b/lib/ontologies_linked_data/media_types.rb @@ -3,8 +3,11 @@ module MediaTypes HTML = :html JSON = :json JSONP = :jsonp + JSONLD = :jsonld XML = :xml + RDF_XML = :rdf_xml TURTLE = :turtle + NTRIPLES = :ntriples DEFAULT = JSON def self.all @@ -55,4 +58,4 @@ def self.supported_type?(type) } end -end \ No newline at end of file +end diff --git a/lib/ontologies_linked_data/models/class.rb b/lib/ontologies_linked_data/models/class.rb index c5fb20d90..754b6cc68 100644 --- a/lib/ontologies_linked_data/models/class.rb +++ b/lib/ontologies_linked_data/models/class.rb @@ -118,6 +118,66 @@ def self.urn_id(acronym,classId) cache_segment_keys [:class] cache_load submission: [ontology: [:acronym]] + # Index settings + def self.index_schema(schema_generator) + schema_generator.add_field(:prefLabel, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:synonym, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:notation, 'string_ci', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:oboId, 'string_ci', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:idAcronymMatch, 'boolean', indexed: true, stored: true, multi_valued: false, default: false) + schema_generator.add_field(:definition, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:submissionAcronym, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:parents, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:ontologyType, 'string', indexed: true, stored: true, multi_valued: false) + # schema_generator.add_field(:ontologyType, 'ontologyType', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:ontologyId, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:submissionId, 'pint', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:childCount, 'pint', indexed: true, stored: true, multi_valued: false) + + schema_generator.add_field(:cui, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:semanticType, 'text_general', indexed: true, stored: true, multi_valued: true) + + schema_generator.add_field(:property, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:propertyRaw, 'text_general', indexed: false, stored: true, multi_valued: false) + + schema_generator.add_field(:obsolete, 'boolean', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:provisional, 'boolean', indexed: true, stored: true, multi_valued: false) + + # Copy fields for term search + schema_generator.add_copy_field('notation', '_text_') + schema_generator.add_copy_field('oboId', '_text_') + + %w[prefLabel synonym].each do |field| + schema_generator.add_field("#{field}Exact", 'string', indexed: true, stored: false, multi_valued: true) + schema_generator.add_field("#{field}Suggest", 'text_suggest', indexed: true, stored: false, multi_valued: true, omit_norms: true) + schema_generator.add_field("#{field}SuggestEdge", 'text_suggest_edge', indexed: true, stored: false, multi_valued: true) + schema_generator.add_field("#{field}SuggestNgram", 'text_suggest_ngram', indexed: true, stored: false, multi_valued: true, omit_norms: true) + + schema_generator.add_copy_field(field, '_text_') + schema_generator.add_copy_field(field, "#{field}Exact") + schema_generator.add_copy_field(field, "#{field}Suggest") + schema_generator.add_copy_field(field, "#{field}SuggestEdge") + schema_generator.add_copy_field(field, "#{field}SuggestNgram") + + schema_generator.add_dynamic_field("#{field}_*", 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_dynamic_field("#{field}Exact_*", 'string', indexed: true, stored: false, multi_valued: true) + schema_generator.add_dynamic_field("#{field}Suggest_*", 'text_suggest', indexed: true, stored: false, multi_valued: true, omit_norms: true) + schema_generator.add_dynamic_field("#{field}SuggestEdge_*", 'text_suggest_edge', indexed: true, stored: false, multi_valued: true) + schema_generator.add_dynamic_field("#{field}SuggestNgram_*", 'text_suggest_ngram', indexed: true, stored: false, multi_valued: true, omit_norms: true) + + schema_generator.add_copy_field("#{field}_*", "#{field}Exact_*") + schema_generator.add_copy_field("#{field}_*", "#{field}Suggest_*") + schema_generator.add_copy_field("#{field}_*", "#{field}SuggestEdge_*") + schema_generator.add_copy_field("#{field}_*", "#{field}SuggestNgram_*") + end + + schema_generator.add_dynamic_field('definition_*', 'text_general', indexed: true, stored: true, multi_valued: true) + end + + enable_indexing(:term_search_core1) do |schema_generator| + index_schema(schema_generator) + end + def self.tree_view_property(*args) submission = args.first unless submission.loaded_attributes.include?(:hasOntologyLanguage) @@ -203,14 +263,15 @@ def index_doc(to_set=nil) path_ids.select! { |x| !x["owl#Thing"] } doc[:parents] = path_ids rescue Exception => e - doc[:parents] = Set.new + doc[:parents] = [] puts "Exception getting paths to root for search for #{class_id}: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}" end acronym = self.submission.ontology.acronym + self.submission.ontology.bring(:ontologyType) if self.submission.ontology.bring?(:ontologyType) doc[:ontologyId] = self.submission.id.to_s - doc[:submissionAcronym] = self.submission.ontology.acronym + doc[:submissionAcronym] = acronym doc[:submissionId] = self.submission.submissionId doc[:ontologyType] = self.submission.ontology.ontologyType.get_code_from_id doc[:obsolete] = self.obsolete.to_s @@ -227,7 +288,10 @@ def index_doc(to_set=nil) if cur_val.is_a?(Hash) # Multi language if multi_language_fields.include?(att) doc[att] = cur_val.values.flatten # index all values of each language - cur_val.each { |lang, values| doc["#{att}_#{lang}".to_sym] = values } # index values per language + cur_val.each do |lang, values| + lang_key = lang.to_s.gsub('@', '') + doc["#{att}_#{lang_key}".to_sym] = values + end # index values per language else doc[att] = cur_val.values.flatten.first end @@ -422,8 +486,8 @@ def retrieve_descendants(page=nil, size=nil) total_size = ids.length if !page.nil? ids = ids.to_a.sort - rstart = (page -1) * size - rend = (page * size) -1 + rstart = (page - 1) * size + rend = (page * size) - 1 ids = ids[rstart..rend] end ids.map! { |x| RDF::URI.new(x) } diff --git a/lib/ontologies_linked_data/models/concerns/parse_diff_file.rb b/lib/ontologies_linked_data/models/concerns/parse_diff_file.rb new file mode 100644 index 000000000..b0ab359d9 --- /dev/null +++ b/lib/ontologies_linked_data/models/concerns/parse_diff_file.rb @@ -0,0 +1,89 @@ +require 'libxml' + +module LinkedData + module Concerns + module SubmissionDiffParser + + class DiffReport + attr_accessor :summary, :changed_classes, :new_classes, :deleted_classes + + def initialize(summary, changed_classes, new_classes, deleted_classes) + @summary = summary + @changed_classes = changed_classes + @new_classes = new_classes + @deleted_classes = deleted_classes + end + end + + class DiffSummary + attr_accessor :number_changed_classes, :number_new_classes, :number_deleted_classes + + def initialize(number_changed_classes, number_new_classes, number_deleted_classes) + @number_changed_classes = number_changed_classes + @number_new_classes = number_new_classes + @number_deleted_classes = number_deleted_classes + end + end + + class ChangedClass + attr_accessor :class_iri, :class_labels, :new_axioms, :new_annotations, :deleted_annotations, :deleted_axioms + + def initialize(class_iri, class_labels, new_axioms, new_annotations, deleted_axioms, deleted_annotations) + @class_iri = class_iri + @class_labels = class_labels + @new_axioms = new_axioms + @deleted_axioms = deleted_axioms + @new_annotations = new_annotations + @deleted_annotations = deleted_annotations + end + end + + class NewClass < ChangedClass; end + + class DeletedClass < ChangedClass; end + + def parse_diff_report(xml_file = self.diffFilePath) + parser = LibXML::XML::Parser.file(xml_file) + doc = parser.parse + + # Parse summary + summary = doc.find_first('//diffSummary') + diff_summary = DiffSummary.new( + summary.find_first('numberChangedClasses').content.to_i, + summary.find_first('numberNewClasses').content.to_i, + summary.find_first('numberDeletedClasses').content.to_i + ) + + # Parse changed classes + changed_classes = doc.find('//changedClasses/changedClass').map do |node| + extract_changes_details ChangedClass, node + end + + # Parse new classes + new_classes = doc.find('//newClasses/newClass').map do |node| + extract_changes_details NewClass, node + end + + # Parse deleted classes + deleted_classes = doc.find('//deletedClasses/deletedClass').map do |node| + extract_changes_details DeletedClass, node + end + + # Create the DiffReport object + DiffReport.new(diff_summary, changed_classes, new_classes, deleted_classes) + end + + def extract_changes_details(klass, node) + class_iri = node.find_first('classIRI').content.strip + class_labels = node.find('classLabel').map(&:content) + new_axioms = node.find('newAxiom').map(&:content) + new_annotations = node.find('newAnnotation').map(&:content) + deleted_axioms = node.find('deletedAxiom').map(&:content) + deleted_annotations = node.find('deletedAnnotation').map(&:content) + + + klass.new(class_iri, class_labels, new_axioms, new_annotations, deleted_annotations, deleted_axioms) + end + end + end +end diff --git a/lib/ontologies_linked_data/models/concerns/submission_process.rb b/lib/ontologies_linked_data/models/concerns/submission_process.rb index fa19df400..240e6a5c8 100644 --- a/lib/ontologies_linked_data/models/concerns/submission_process.rb +++ b/lib/ontologies_linked_data/models/concerns/submission_process.rb @@ -2,10 +2,22 @@ module LinkedData module Concerns module SubmissionProcessable - def process_submission(logger, options={}) + def process_submission(logger, options = {}) LinkedData::Services::OntologyProcessor.new(self).process(logger, options) end + def generate_missing_labels(logger) + LinkedData::Services::GenerateMissingLabels.new(self).process(logger, file_path: self.master_file_path) + end + + def generate_obsolete_classes(logger) + LinkedData::Services::ObsoleteClassesGenerator.new(self).process(logger, file_path: self.master_file_path) + end + + def extract_metadata(logger, options = {}) + LinkedData::Services::SubmissionMetadataExtractor.new(self).process(logger, options) + end + def diff(logger, older) LinkedData::Services::SubmissionDiffGenerator.new(self).diff(logger, older) end @@ -14,7 +26,11 @@ def generate_diff(logger) LinkedData::Services::SubmissionDiffGenerator.new(self).process(logger) end - def index(logger, commit: true, optimize: true) + def index_all(logger, commit: true) + LinkedData::Services::OntologySubmissionAllDataIndexer.new(self).process(logger, commit: commit) + end + + def index_terms(logger, commit: true, optimize: true) LinkedData::Services::OntologySubmissionIndexer.new(self).process(logger, commit: commit, optimize: optimize) end @@ -22,8 +38,8 @@ def index_properties(logger, commit: true, optimize: true) LinkedData::Services::SubmissionPropertiesIndexer.new(self).process(logger, commit: commit, optimize: optimize) end - def archive - LinkedData::Services::OntologySubmissionArchiver.new(self ).process + def archive(force: false) + LinkedData::Services::OntologySubmissionArchiver.new(self).process(force: force) end def generate_rdf(logger, reasoning: true) @@ -36,5 +52,4 @@ def generate_metrics(logger) end end -end - +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/models/contact.rb b/lib/ontologies_linked_data/models/contact.rb index 2306135dc..387c9f38c 100644 --- a/lib/ontologies_linked_data/models/contact.rb +++ b/lib/ontologies_linked_data/models/contact.rb @@ -6,6 +6,13 @@ class Contact < LinkedData::Models::Base attribute :email, enforce: [:existence, :email] embedded true + + def embedded_doc + bring(:name) if bring?(:name) + bring(:email) if bring?(:email) + + "#{self.name} | #{self.email}" + end end end end diff --git a/lib/ontologies_linked_data/models/external_class.rb b/lib/ontologies_linked_data/models/external_class.rb new file mode 100644 index 000000000..3a0d5ffe2 --- /dev/null +++ b/lib/ontologies_linked_data/models/external_class.rb @@ -0,0 +1,40 @@ +module LinkedData + module Models + class ExternalClass + include LinkedData::Hypermedia::Resource + # For class mapped to internal class that are outside any BioPortal appliance + # We just generate a link to self class and a link to the external ontology + + attr_reader :id, :ontology, :type_uri + attr_accessor :prefLabel + + serialize_never :id, :ontology, :type_uri + + link_to LinkedData::Hypermedia::Link.new("self", lambda {|ec| ec.id.to_s}, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("ontology", lambda {|ec| ec.ontology.to_s}, Goo.vocabulary["Ontology"]) + + def initialize(id, ontology) + @id = id + @ontology = RDF::URI.new(CGI.unescape(ontology.to_s)) + @type_uri = RDF::URI.new("http://www.w3.org/2002/07/owl#Class") + end + + def getPrefLabel + # take the last part of the URL to generate the prefLabel (the one after the last #, or if not after the last /) + if id.include? "#" + @prefLabel = id.split("#")[-1] + else + @prefLabel = id.split("/")[-1] + end + end + + def self.graph_uri + RDF::URI.new("http://data.bioontology.org/metadata/ExternalMappings") + end + def self.url_param_str + # a little string to get external mappings in URL parameters + RDF::URI.new("mappings:external") + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/models/interportal_class.rb b/lib/ontologies_linked_data/models/interportal_class.rb new file mode 100644 index 000000000..fbd90ccd7 --- /dev/null +++ b/lib/ontologies_linked_data/models/interportal_class.rb @@ -0,0 +1,80 @@ +module LinkedData + module Models + class InterportalClass + include LinkedData::Hypermedia::Resource + # For class mapped to internal class that are inside another BioPortal appliance + # We are generating the same link than a normal class but pointing to the other appliance + + attr_reader :id, :ontology, :type_uri, :source, :ui_link, :self + attr_accessor :prefLabel + + serialize_never :id, :ontology, :type_uri, :source, :ui_link, :self + + link_to LinkedData::Hypermedia::Link.new("self", lambda { |ec| "#{ec.self.to_s}" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("ontology", lambda { |ec| ec.ontology.to_s }, Goo.vocabulary["Ontology"]), + LinkedData::Hypermedia::Link.new("children", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/children" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("parents", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/parents" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("descendants", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/descendants" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("ancestors", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/ancestors" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("tree", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/tree" }, "http://www.w3.org/2002/07/owl#Class"), + LinkedData::Hypermedia::Link.new("notes", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/notes" }, LinkedData::Models::Note.type_uri), + LinkedData::Hypermedia::Link.new("mappings", lambda { |ec| "#{ec.ontology.to_s}/classes/#{CGI.escape(ec.id.to_s)}/mappings" }, Goo.vocabulary["Mapping"]), + LinkedData::Hypermedia::Link.new("ui", lambda { |ec| ec.ui_link.to_s }, "http://www.w3.org/2002/07/owl#Class") + + def initialize(id, ontology, source) + @id = id + + @ontology, acronym = ontology_url(ontology, source) + @self = "#{@ontology}/classes/#{CGI.escape(id.to_s)}" + @ui_link = "#{LinkedData.settings.interportal_hash[source]["ui"]}/ontologies/#{acronym}?p=classes&conceptid=#{CGI.escape(id)}" + @type_uri = RDF::URI.new("http://www.w3.org/2002/07/owl#Class") + @source = source + end + + def getPrefLabel + # Get the prefLabel from the source bioportal, if error it generates the label from the last part of the URL + begin + json_class = JSON.parse(Net::HTTP.get(URI.parse("#{@self}?apikey=#{LinkedData.settings.interportal_hash[@source]["apikey"]}"))) + @prefLabel = if !json_class["prefLabel"].nil? + json_class["prefLabel"] + else + id.split("/")[-1] + end + rescue + @prefLabel = id.split("/")[-1] + end + end + + def self.graph_uri(acronym) + RDF::URI.new("http://data.bioontology.org/metadata/InterportalMappings/#{acronym}") + end + + def self.graph_base_str + "http://data.bioontology.org/metadata/InterportalMappings" + end + + def self.url_param_str(acronym) + # a little string to get interportal mappings in URL parameters + RDF::URI.new("interportal:#{acronym}") + end + + def self.base_url_param_str + RDF::URI.new("interportal:") + end + + private + + def ontology_url(ontology, source) + url = '' + if ontology =~ URI::DEFAULT_PARSER.make_regexp + url = ontology + else + url = "#{LinkedData.settings.interportal_hash[source]['api']}/ontologies/#{ontology}" + end + acronym = url.split('/').last + [url, acronym] + + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/models/metric.rb b/lib/ontologies_linked_data/models/metric.rb index 102d41f77..f7f107c40 100644 --- a/lib/ontologies_linked_data/models/metric.rb +++ b/lib/ontologies_linked_data/models/metric.rb @@ -51,6 +51,14 @@ def self.metrics_id_generator(m) raise ArgumentError, "Metrics id needs to be set" #return RDF::URI.new(m.submission.id.to_s + "/metrics") end + + def embedded_doc + doc = indexable_object + doc.delete(:resource_model) + doc.delete(:resource_id) + doc.delete(:id) + doc + end end end end diff --git a/lib/ontologies_linked_data/models/mod/hydra_page.rb b/lib/ontologies_linked_data/models/mod/hydra_page.rb new file mode 100644 index 000000000..877b4c9d9 --- /dev/null +++ b/lib/ontologies_linked_data/models/mod/hydra_page.rb @@ -0,0 +1,59 @@ +module LinkedData + module Models + class HydraPage < Goo::Base::Page + + def convert_hydra_page(options, &block) + { + '@id': get_request_path(options), + '@type': 'hydra:Collection', + totalItems: self.aggregate, + itemsPerPage: self.size, + view: generate_hydra_page_view(options, self.page_number, self.total_pages), + member: map { |item| item.to_flex_hash(options, &block) } + } + end + + def self.generate_hydra_context + { + 'hydra': 'http://www.w3.org/ns/hydra/core#', + 'Collection': 'hydra:Collection', + 'member': 'hydra:member', + 'totalItems': 'hydra:totalItems', + 'itemsPerPage': 'hydra:itemsPerPage', + 'view': 'hydra:view', + 'firstPage': 'hydra:first', + 'lastPage': 'hydra:last', + 'previousPage': 'hydra:previous', + 'nextPage': 'hydra:next', + } + end + + private + + def generate_hydra_page_view(options, page, page_count) + request_path = get_request_path(options) + params = options[:request] ? options[:request].params.dup : {} + + build_url = ->(page_number) { + query = Rack::Utils.build_nested_query(params.merge("page" => page_number.to_s)) + request_path ? "#{request_path}?#{query}" : "?#{query}" + } + + { + "@id": build_url.call(page), + "@type": "hydra:PartialCollectionView", + firstPage: build_url.call(1), + previousPage: page > 1 ? build_url.call(page - 1) : nil, + nextPage: page < page_count ? build_url.call(page + 1) : nil, + lastPage: page_count != 0 ? build_url.call(page_count) : build_url.call(1) + } + end + + def get_request_path(options) + request_path = options[:request] ? "#{LinkedData.settings.rest_url_prefix.chomp("/")}#{options[:request].path}" : nil + request_path + end + + end + end +end diff --git a/lib/ontologies_linked_data/models/mod/mod_base.rb b/lib/ontologies_linked_data/models/mod/mod_base.rb new file mode 100644 index 000000000..5208e8bbe --- /dev/null +++ b/lib/ontologies_linked_data/models/mod/mod_base.rb @@ -0,0 +1,7 @@ +module LinkedData + module Models + class ModBase < LinkedData::Models::Base + end + end +end + diff --git a/lib/ontologies_linked_data/models/ontology.rb b/lib/ontologies_linked_data/models/ontology.rb index 0ce2ae432..9de084068 100644 --- a/lib/ontologies_linked_data/models/ontology.rb +++ b/lib/ontologies_linked_data/models/ontology.rb @@ -24,8 +24,8 @@ class OntologyAnalyticsError < StandardError; end model :ontology, :name_with => :acronym attribute :acronym, namespace: :omv, - enforce: [:unique, :existence, lambda { |inst,attr| validate_acronym(inst,attr) } ] - attribute :name, :namespace => :omv, enforce: [:unique, :existence, :safe_text_256] + enforce: [:unique, :existence, lambda { |inst,attr| validate_acronym(inst,attr) } ], fuzzy_search: true + attribute :name, :namespace => :omv, enforce: [:unique, :existence, :safe_text_256], fuzzy_search: true attribute :submissions, inverse: { on: :ontology_submission, attribute: :ontology } attribute :projects, @@ -49,7 +49,7 @@ class OntologyAnalyticsError < StandardError; end attribute :acl, enforce: [:list, :user] - attribute :viewOf, enforce: [:ontology] + attribute :viewOf, enforce: [:ontology], onUpdate: :update_submissions_has_part attribute :views, :inverse => { on: :ontology, attribute: :viewOf } attribute :ontologyType, enforce: [:ontology_type], default: lambda { |record| LinkedData::Models::OntologyType.find("ONTOLOGY").include(:code).first } @@ -86,6 +86,10 @@ class OntologyAnalyticsError < StandardError; end # Cache cache_timeout 3600 + enable_indexing(:ontology_metadata) + + after_save :index_latest_submission + def self.validate_acronym(inst, attr) inst.bring(attr) if inst.bring?(attr) acronym = inst.send(attr) @@ -113,6 +117,53 @@ def self.validate_acronym(inst, attr) return errors.flatten end + def update_submissions_has_part(inst, attr) + inst.bring :viewOf if inst.bring?(:viewOf) + + target_ontology = inst.viewOf + + if target_ontology.nil? + previous_value = inst.previous_values ? inst.previous_values[attr] : nil + return if previous_value.nil? + + action = :remove + target_ontology = previous_value + else + action = :append + end + + sub = target_ontology.latest_submission || target_ontology.bring(:submissions) && target_ontology.submissions.last + + return if sub.nil? + + sub.bring :hasPart if sub.bring?(:hasPart) + + parts = sub.hasPart.dup || [] + changed = false + if action.eql?(:append) + unless parts.include?(self.id) + changed = true + parts << self.id + end + elsif action.eql?(:remove) + if parts.include?(self.id) + changed = true + parts.delete(self.id) + sub.class.model_settings[:attributes][:hasPart][:enforce].delete(:include_ontology_views) #disable validator + end + end + + return unless changed + + sub.bring_remaining + sub.hasPart = parts + sub.save if sub.valid? + + return unless changed && action.eql?(:remove) + + sub.class.model_settings[:attributes][:hasPart][:enforce].append(:include_ontology_views) + end + def latest_submission(options = {}) self.bring(:acronym) if self.bring?(:acronym) submission_id = highest_submission_id(options) @@ -160,7 +211,7 @@ def highest_submission_id(options = {}) begin subs = self.submissions - rescue Exception => e + rescue Exception i = 0 num_calls = LinkedData.settings.num_retries_4store subs = nil @@ -251,7 +302,7 @@ def property_roots(sub=nil, extra_include=[]) LinkedData::Models::OntologyProperty.sort_properties(all_roots) end - def property(prop_id, sub=nil) + def property(prop_id, sub=nil, display_all_attributes: false) p = nil sub ||= latest_submission(status: [:rdf]) self.bring(:acronym) if self.bring?(:acronym) @@ -259,9 +310,10 @@ def property(prop_id, sub=nil) prop_classes = [LinkedData::Models::ObjectProperty, LinkedData::Models::DatatypeProperty, LinkedData::Models::AnnotationProperty] prop_classes.each do |c| - p = c.find(prop_id).in(sub).include(:label, :definition, :parents).first + p = c.find(prop_id).in(sub).include(:label, :definition, :parents,:domain, :range).first unless p.nil? + p.bring(:unmapped) if display_all_attributes p.load_has_children parents = p.parents.nil? ? [] : p.parents.dup c.in(sub).models(parents).include(:label, :definition).all() @@ -394,8 +446,7 @@ def delete(*args) end # remove index entries - unindex(index_commit) - unindex_properties(index_commit) + unindex_all_data(index_commit) # delete all files ontology_dir = File.join(LinkedData.settings.repository_folder, self.acronym.to_s) @@ -417,19 +468,43 @@ def save(*args) self end - def unindex(commit=true) + def index_latest_submission + last_s = latest_submission(status: :any) + return if last_s.nil? + + last_s.ontology = self + last_s.index_update([:ontology]) + end + + def unindex_all_data(commit=true) unindex_by_acronym(commit) + unindex_properties(commit) + end + + def embedded_doc + self.administeredBy.map{|x| x.bring_remaining} + doc = indexable_object + doc.delete(:id) + doc.delete(:resource_id) + doc.delete('ontology_viewOf_resource_model_t') + doc['ontology_viewOf_t'] = self.viewOf.id.to_s unless self.viewOf.nil? + doc[:resource_model_t] = doc.delete(:resource_model) + doc end def unindex_properties(commit=true) - unindex_by_acronym(commit, :property) + self.bring(:acronym) if self.bring?(:acronym) + query = "submissionAcronym:#{acronym}" + OntologyProperty.unindexByQuery(query) + OntologyProperty.indexCommit(nil) if commit end - def unindex_by_acronym(commit=true, connection_name=:main) + def unindex_by_acronym(commit=true) self.bring(:acronym) if self.bring?(:acronym) query = "submissionAcronym:#{acronym}" - Ontology.unindexByQuery(query, connection_name) - Ontology.indexCommit(nil, connection_name) if commit + Class.unindexByQuery(query) + Class.indexCommit(nil) if commit + #OntologySubmission.clear_indexed_content(acronym) end def restricted? diff --git a/lib/ontologies_linked_data/models/ontology_format.rb b/lib/ontologies_linked_data/models/ontology_format.rb index c582ef90f..926c8c60b 100644 --- a/lib/ontologies_linked_data/models/ontology_format.rb +++ b/lib/ontologies_linked_data/models/ontology_format.rb @@ -39,14 +39,14 @@ def tree_property return Goo.vocabulary(:metadata)[:treeView] end if skos? - return RDF::SKOS[:broader] + return RDF::Vocab::SKOS[:broader] end return RDF::RDFS[:subClassOf] end def class_type if skos? - return RDF::SKOS[:Concept] + return RDF::Vocab::SKOS[:Concept] end return RDF::OWL[:Class] end diff --git a/lib/ontologies_linked_data/models/ontology_submission.rb b/lib/ontologies_linked_data/models/ontology_submission.rb index 83810a2a6..1397de795 100644 --- a/lib/ontologies_linked_data/models/ontology_submission.rb +++ b/lib/ontologies_linked_data/models/ontology_submission.rb @@ -13,10 +13,10 @@ module Models class OntologySubmission < LinkedData::Models::Base include LinkedData::Concerns::SubmissionProcessable - include LinkedData::Concerns::OntologySubmission::MetadataExtractor include LinkedData::Concerns::OntologySubmission::Validators + include LinkedData::Concerns::OntologySubmission::UpdateCallbacks extend LinkedData::Concerns::OntologySubmission::DefaultCallbacks - + include LinkedData::Concerns::SubmissionDiffParser include SKOS::ConceptSchemes include SKOS::RootsFetcher @@ -128,8 +128,7 @@ class OntologySubmission < LinkedData::Models::Base attribute :uriLookupEndpoint, namespace: :void, type: :uri, default: -> (s) { uri_lookup_default(s) } attribute :openSearchDescription, namespace: :void, type: :uri, default: -> (s) { open_search_default(s) } attribute :source, namespace: :dct, type: :list - attribute :endpoint, namespace: :sd, type: %i[uri list], - default: ->(s) { default_sparql_endpoint(s) } + attribute :endpoint, namespace: :sd, type: %i[uri list], default: ->(s) { default_sparql_endpoint(s) } attribute :includedInDataCatalog, namespace: :schema, type: %i[list uri] # Relations @@ -181,26 +180,11 @@ class OntologySubmission < LinkedData::Models::Base # System-controlled attributes that should not be set by API clients system_controlled :submissionId, :uploadFilePath, :diffFilePath, :missingImports - def self.agents_attrs - return [] #TODO implement agent separately - %i[hasCreator publisher copyrightHolder hasContributor - translator endorsedBy fundedBy curatedBy] - end - # Hypermedia settings - embed *%i[contact ontology metrics] + agents_attrs + embed *(%i[contact ontology metrics]) def self.embed_values_hash - out = { - submissionStatus: [:code], hasOntologyLanguage: [:acronym] - } - - # TODO implement agents separately - # agent_attributes = LinkedData::Models::Agent.goo_attrs_to_load + - # [identifiers: LinkedData::Models::AgentIdentifier.goo_attrs_to_load, affiliations: LinkedData::Models::Agent.goo_attrs_to_load] - # - # agents_attrs.each { |k| out[k] = agent_attributes } - out + { submissionStatus: [:code], hasOntologyLanguage: [:acronym] } end embed_values self.embed_values_hash @@ -230,6 +214,8 @@ def self.embed_values_hash read_restriction_based_on lambda { |sub| sub.ontology } access_control_load ontology: [:administeredBy, :acl, :viewingRestriction] + enable_indexing(:ontology_metadata) + def initialize(*args) super(*args) @mutex = Mutex.new @@ -240,7 +226,7 @@ def synchronize(&block) end def URI=(value) - self.uri = value + self.uri = value end def URI @@ -256,13 +242,31 @@ def self.ontology_link(m) begin m.ontology.bring(:acronym) if m.ontology.bring?(:acronym) ontology_link = "ontologies/#{m.ontology.acronym}" - rescue Exception => e + rescue Exception ontology_link = "" end end ontology_link end + # Override the bring_remaining method from Goo::Base::Resource : https://github.com/ncbo/goo/blob/master/lib/goo/base/resource.rb#L383 + # Because the old way to query the 4store was not working when lots of attributes + # Now it is querying attributes 5 by 5 (way faster than 1 by 1) + def bring_remaining + to_bring = [] + i = 0 + self.class.attributes.each do |attr| + to_bring << attr if self.bring?(attr) + if i == 5 + self.bring(*to_bring) + to_bring = [] + i = 0 + end + i = i + 1 + end + self.bring(*to_bring) + end + def self.segment_instance(sub) sub.bring(:ontology) unless sub.loaded_attributes.include?(:ontology) sub.ontology.bring(:acronym) unless sub.ontology.loaded_attributes.include?(:acronym) @@ -315,6 +319,18 @@ def self.copy_file_repository(acronym, submission_id, src, filename = nil) dst end + def self.clear_indexed_content(ontology_acronym, logger=nil) + logger ||= LinkedData::Parser.logger || Logger.new($stderr) + conn = Goo.init_search_connection(:ontology_data) + begin + conn.delete_by_query("ontology_t:\"#{ontology_acronym}\"") + rescue StandardError => e + logger.error("Unable to clear search index for #{ontology_acronym} - #{e.class}: #{e.message}\n" + e.backtrace.join("\n\t")) + logger.flush + end + conn + end + def valid? valid_result = super return false unless valid_result @@ -348,7 +364,7 @@ def sanity_check begin sum_only = self.ontology.summaryOnly - rescue Exception => e + rescue Exception i = 0 num_calls = LinkedData.settings.num_retries_4store sum_only = nil @@ -362,7 +378,7 @@ def sanity_check self.ontology.bring(:summaryOnly) sum_only = self.ontology.summaryOnly puts "Success getting summaryOnly for #{self.id.to_s} after retrying #{i} times..." - rescue Exception => e1 + rescue Exception sum_only = nil if i == num_calls @@ -495,9 +511,19 @@ def unzip_submission(logger) # Set master file name automatically if there is only one file if extracted.length == 1 && self.masterFileName.nil? - self.masterFileName = extracted.first.name + first = extracted.first + + self.masterFileName = + if first.respond_to?(:name) + first.name + elsif first.is_a?(Hash) + first[:name] || first["name"] + else + first.to_s + end self.save end + self.masterFileName = File.basename(self.masterFileName.to_s) if logger logger.info("Files extracted from zip #{extracted}") @@ -559,7 +585,7 @@ def metrics_from_file(logger=nil) begin metrics = CSV.read(m_path) - rescue Exception => e + rescue Exception logger.error("Unable to find metrics file: #{m_path}") logger.flush end @@ -808,18 +834,6 @@ def ontology_uri RDF::URI.new(self.uri) end - - - - - - - - - - - - def roots_sorted(extra_include=nil) classes = roots(extra_include) LinkedData::Models::Class.sort_classes(classes) @@ -921,7 +935,7 @@ def check_ftp_file(uri) ftp.login begin file_exists = ftp.size(uri.path) > 0 - rescue Exception => e + rescue Exception # Check using another method path = uri.path.split("/") filename = path.pop diff --git a/lib/ontologies_linked_data/models/properties/annotation_property.rb b/lib/ontologies_linked_data/models/properties/annotation_property.rb index b071d09fb..7e2167177 100644 --- a/lib/ontologies_linked_data/models/properties/annotation_property.rb +++ b/lib/ontologies_linked_data/models/properties/annotation_property.rb @@ -18,11 +18,12 @@ class AnnotationProperty < LinkedData::Models::OntologyProperty attribute :children, namespace: :rdfs, inverse: { on: :annotation_property, :attribute => :parents } attribute :ancestors, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_ancestors attribute :descendants, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_descendants - # attribute :domain - # attribute :range + attribute :domain, namespace: :rdfs + attribute :range, namespace: :rdfs - serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren # some of these attributes are used in Search (not shown out of context) + serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren,:domain, :range # some of these attributes are used in Search (not shown out of context) aggregates childrenCount: [:count, :children] + serialize_methods :properties # this command allows the children to be serialized in the output embed :children @@ -34,6 +35,10 @@ class AnnotationProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("ancestors", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/ancestors"}, self.uri_type), LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) + + enable_indexing(:prop_search_core1, :property) do |schema_generator| + index_schema(schema_generator) + end end end diff --git a/lib/ontologies_linked_data/models/properties/datatype_property.rb b/lib/ontologies_linked_data/models/properties/datatype_property.rb index 1974bdb2d..34789dc1b 100644 --- a/lib/ontologies_linked_data/models/properties/datatype_property.rb +++ b/lib/ontologies_linked_data/models/properties/datatype_property.rb @@ -18,11 +18,12 @@ class DatatypeProperty < LinkedData::Models::OntologyProperty attribute :children, namespace: :rdfs, inverse: { on: :datatype_property, :attribute => :parents } attribute :ancestors, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_ancestors attribute :descendants, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_descendants - # attribute :domain - # attribute :range + attribute :domain, namespace: :rdfs + attribute :range, namespace: :rdfs - serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren # some of these attributes are used in Search (not shown out of context) + serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren, :domain, :range # some of these attributes are used in Search (not shown out of context) aggregates childrenCount: [:count, :children] + serialize_methods :properties # this command allows the children to be serialized in the output embed :children @@ -34,6 +35,10 @@ class DatatypeProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("ancestors", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/ancestors"}, self.uri_type), LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) + + enable_indexing(:prop_search_core1, :property) do |schema_generator| + index_schema(schema_generator) + end end end diff --git a/lib/ontologies_linked_data/models/properties/object_property.rb b/lib/ontologies_linked_data/models/properties/object_property.rb index 8abbc52fb..3b96216dc 100644 --- a/lib/ontologies_linked_data/models/properties/object_property.rb +++ b/lib/ontologies_linked_data/models/properties/object_property.rb @@ -18,11 +18,12 @@ class ObjectProperty < LinkedData::Models::OntologyProperty attribute :children, namespace: :rdfs, inverse: { on: :object_property, :attribute => :parents } attribute :ancestors, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_ancestors attribute :descendants, namespace: :rdfs, property: :subPropertyOf, handler: :retrieve_descendants - # attribute :domain - # attribute :range + attribute :domain, namespace: :rdfs + attribute :range, namespace: :rdfs - serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren # some of these attributes are used in Search (not shown out of context) + serialize_default :label, :labelGenerated, :definition, :matchType, :ontologyType, :propertyType, :parents, :children, :hasChildren, :domain, :range # some of these attributes are used in Search (not shown out of context) aggregates childrenCount: [:count, :children] + serialize_methods :properties # this command allows the children to be serialized in the output embed :children @@ -34,6 +35,10 @@ class ObjectProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("ancestors", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/ancestors"}, self.uri_type), LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) + + enable_indexing(:prop_search_core1, :property) do |schema_generator| + index_schema(schema_generator) + end end end diff --git a/lib/ontologies_linked_data/models/properties/ontology_property.rb b/lib/ontologies_linked_data/models/properties/ontology_property.rb index 1e9ced842..7aaa4f541 100644 --- a/lib/ontologies_linked_data/models/properties/ontology_property.rb +++ b/lib/ontologies_linked_data/models/properties/ontology_property.rb @@ -3,6 +3,36 @@ module LinkedData module Models class OntologyProperty < LinkedData::Models::Base + model :ontology_property, name_with: ->(p) { uuid_uri_generator(p) } + + def self.index_schema(schema_generator) + schema_generator.add_field(:label, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:labelGenerated, 'text_general', indexed: true, stored: true, multi_valued: true) + + schema_generator.add_field(:definition, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:submissionAcronym, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:parents, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:ontologyType, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:propertyType, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:ontologyId, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:submissionId, 'pint', indexed: true, stored: true, multi_valued: false) + + %i[label labelGenerated].each do |field| + schema_generator.add_copy_field(field, '_text_') + schema_generator.add_copy_field(field, "#{field}Exact") + schema_generator.add_copy_field(field, "#{field}Suggest") + schema_generator.add_copy_field(field, "#{field}SuggestEdge") + schema_generator.add_copy_field(field, "#{field}SuggestNgram") + end + end + + enable_indexing(:prop_search_core1, :property) do |schema_generator| + index_schema(schema_generator) + end + + def properties + self.unmapped + end def retrieve_ancestors retrieve_ancestors_descendants(:ancestors) @@ -131,9 +161,9 @@ def has_children_query(class_id, submission_id) pattern = "?c <#{property_tree.to_s}> <#{safe_class_id_uri}> . " query = < { - #{pattern} -} + GRAPH <#{submission_id}> { + #{pattern} + } } LIMIT 1 eos @@ -234,7 +264,7 @@ def index_doc(to_set=nil) } all_attrs = self.to_hash - std = [:id, :label, :definition, :parents] + std = %i[id label definition parents] std.each do |att| cur_val = all_attrs[att] @@ -288,7 +318,7 @@ def traverse_path_to_root(parents, paths, path_i, tree=false, top_property=nil) rec_i = recursions[i] path = paths[rec_i] p = path.last - p.bring(parents: [:label, :definition]) if p.bring?(:parents) + p.bring(parents: %i[label definition]) if p.bring?(:parents) unless p.loaded_attributes.include?(:parents) # fail safely @@ -313,7 +343,7 @@ def self.ontology_link(m) end def self.partially_load_children(models, threshold, submission) - ld = [:label, :definition] + ld = %i[label definition] single_load = [] query = self.in(submission).models(models) query.aggregate(:count, :children).all @@ -394,8 +424,7 @@ def self.compare_properties(prop_a, prop_b) [label_a.downcase] <=> [label_b.downcase] end - end - end + end diff --git a/lib/ontologies_linked_data/models/provisional_class.rb b/lib/ontologies_linked_data/models/provisional_class.rb index 4b872e13a..a40b936b8 100644 --- a/lib/ontologies_linked_data/models/provisional_class.rb +++ b/lib/ontologies_linked_data/models/provisional_class.rb @@ -35,11 +35,15 @@ class ProvisionalClass < LinkedData::Models::Base else "" end - rescue Exception => e + rescue Exception "" end }, Goo.vocabulary["Ontology"]) + enable_indexing(:term_search_core1) do |schema_generator| + Class.index_schema(schema_generator) + end + def index_id() self.bring(:ontology) if self.bring?(:ontology) return nil unless self.ontology @@ -143,38 +147,6 @@ def append_if_not_there_already(path, r) true end - def index() - if index_id - unindex - super - LinkedData::Models::Ontology.indexCommit - end - end - - def unindex() - ind_id = index_id - - if ind_id - query = "id:#{solr_escape(ind_id)}" - LinkedData::Models::Ontology.unindexByQuery(query) - LinkedData::Models::Ontology.indexCommit - end - end - - ## - # Override save to allow indexing - def save(*args) - super(*args) - index - self - end - - def delete(*args) - # remove index entries - unindex - super(*args) - end - def solr_escape(text) RSolr.solr_escape(text).gsub(/\s+/,"\\ ") end diff --git a/lib/ontologies_linked_data/models/resource.rb b/lib/ontologies_linked_data/models/resource.rb new file mode 100644 index 000000000..66034d335 --- /dev/null +++ b/lib/ontologies_linked_data/models/resource.rb @@ -0,0 +1,189 @@ +require 'rdf/raptor' + +module LinkedData + module Models + + class Resource + + def initialize(graph, id) + @id = id + @graph = graph + @hash = fetch_related_triples(graph, id) + end + + def to_hash + @hash.dup + end + + def to_object + hashes = self.to_hash + class_name = "GeneratedModel_#{Time.now.to_i}_#{rand(10000..99999)}" + model_schema = ::Class.new(LinkedData::Models::Base) + Object.const_set(class_name, model_schema) + + model_schema.model(:resource, name_with: :id, rdf_type: lambda { |*_x| self.to_hash[Goo.namespaces[:rdf][:type].to_s] }) + values_hash = {} + hashes.each do |predicate, value| + namespace, attr = namespace_predicate(predicate) + next if namespace.nil? + + values = Array(value).map do |v| + if v.is_a?(Hash) + Struct.new(*v.keys.map { |k| namespace_predicate(k)[1].to_sym }.compact).new(*v.values) + else + v.is_a?(RDF::URI) ? v.to_s : v.object + end + end.compact + + model_schema.attribute(attr.to_sym, property: namespace.to_s, enforce: get_type(value)) + values_hash[attr.to_sym] = value.is_a?(Array) ? values : values.first + end + + values_hash[:id] = hashes['id'] + model_schema.new(values_hash) + end + + def to_json + LinkedData::Serializers.serialize(to_hash, LinkedData::MediaTypes::JSONLD, namespaces) + end + + def to_xml + LinkedData::Serializers.serialize(to_hash, LinkedData::MediaTypes::RDF_XML, namespaces) + end + + def to_ntriples + LinkedData::Serializers.serialize(to_hash, LinkedData::MediaTypes::NTRIPLES, namespaces) + end + + def to_turtle + LinkedData::Serializers.serialize(to_hash, LinkedData::MediaTypes::TURTLE, namespaces) + end + + def namespaces + prefixes = {} + ns_count = 0 + hash = to_hash + reverse = hash.delete('reverse') + + hash.each do |key, value| + uris = [key] + uris += Array(value).map { |v| v.is_a?(Hash) ? v.to_a.flatten : v }.flatten + prefixes, ns_count = transform_to_prefixes(ns_count, prefixes, uris) + end + + reverse.each { |key, uris| prefixes, ns_count = transform_to_prefixes(ns_count, prefixes, [key] + Array(uris)) } + + prefixes + end + + private + + def transform_to_prefixes(ns_count, prefixes, uris) + uris.each do |uri| + namespace, _ = namespace_predicate(uri) + next if namespace.nil? || prefixes.value?(namespace) + + prefix, prefix_namespace = Goo.namespaces.select { |_k, v| v.to_s.eql?(namespace) }.first + if prefix + prefixes[prefix] = prefix_namespace.to_s + else + prefixes["ns#{ns_count}".to_sym] = namespace + ns_count += 1 + end + end + [prefixes, ns_count] + end + + def fetch_related_triples(graph, id) + direct_fetch_query = Goo.sparql_query_client.select(:predicate, :object) + .from(RDF::URI.new(graph)) + .where([RDF::URI.new(id), :predicate, :object]) + + inverse_fetch_query = Goo.sparql_query_client.select(:subject, :predicate) + .from(RDF::URI.new(graph)) + .where([:subject, :predicate, RDF::URI.new(id)]) + + hashes = { 'id' => RDF::URI.new(id) } + + direct_fetch_query.each_solution do |solution| + predicate = solution[:predicate].to_s + value = solution[:object] + + if value.is_a?(RDF::Node) && Array(hashes[predicate]).none? { |x| x.is_a?(Hash) } + value = fetch_b_nodes_triples(graph, id, solution[:predicate]) + elsif value.is_a?(RDF::Node) + next + end + + hashes[predicate] = hashes[predicate] ? (Array(hashes[predicate]) + Array(value)) : value + end + + hashes['reverse'] = {} + inverse_fetch_query.each_solution do |solution| + subject = solution[:subject].to_s + predicate = solution[:predicate] + + if hashes['reverse'][subject] + if hashes['reverse'][subject].is_a?(Array) + hashes['reverse'][subject] << predicate + else + hashes['reverse'][subject] = [predicate, hashes['reverse'][subject]] + end + else + hashes['reverse'][subject] = predicate + end + + end + + hashes + end + + def fetch_b_nodes_triples(graph, id, predicate) + b_node_fetch_query = Goo.sparql_query_client.select(:b, :predicate, :object) + .from(RDF::URI.new(graph)) + .where( + [RDF::URI.new(id), predicate, :b], + %i[b predicate object] + ) + + b_nodes_hash = {} + b_node_fetch_query.each_solution do |s| + b_node_id = s[:b].to_s + s[:predicate].to_s + s[:object] + if b_nodes_hash[b_node_id] + b_nodes_hash[b_node_id][s[:predicate].to_s] = s[:object] + else + b_nodes_hash[b_node_id] = { s[:predicate].to_s => s[:object] } + end + end + b_nodes_hash.values + end + + def get_type(value) + types = [] + types << :list if value.is_a?(Array) + value = Array(value).first + if value.is_a?(RDF::URI) + types << :uri + elsif value.is_a?(Float) + types << :float + elsif value.is_a?(Integer) + types << :integer + elsif value.to_s.eql?('true') || value.to_s.eql?('false') + types << :boolean + end + types + end + + def namespace_predicate(property_url) + return nil if property_url.is_a?(RDF::Literal) || !URI::DEFAULT_PARSER.make_regexp.match?(property_url) + + regex = %r{^(?.*[/#])(?[^/#]+)$} + match = regex.match(property_url.to_s) + [match[:namespace], match[:id]] if match + end + + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/models/skos/collection.rb b/lib/ontologies_linked_data/models/skos/collection.rb index afc5724e4..535ace488 100644 --- a/lib/ontologies_linked_data/models/skos/collection.rb +++ b/lib/ontologies_linked_data/models/skos/collection.rb @@ -4,7 +4,7 @@ module SKOS class Collection < LinkedData::Models::Base model :collection, name_with: :id, collection: :submission, - namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:Collection] } + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::Vocab::SKOS[:Collection] } attribute :prefLabel, namespace: :skos, enforce: [:existence] attribute :member, namespace: :skos, enforce: [:list, :class] diff --git a/lib/ontologies_linked_data/models/skos/scheme.rb b/lib/ontologies_linked_data/models/skos/scheme.rb index 37e041896..1aca93932 100644 --- a/lib/ontologies_linked_data/models/skos/scheme.rb +++ b/lib/ontologies_linked_data/models/skos/scheme.rb @@ -4,7 +4,7 @@ module SKOS class Scheme < LinkedData::Models::Base model :scheme, name_with: :id, collection: :submission, - namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:ConceptScheme] } + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::Vocab::SKOS[:ConceptScheme] } attribute :prefLabel, namespace: :skos, enforce: [:existence] diff --git a/lib/ontologies_linked_data/models/slice.rb b/lib/ontologies_linked_data/models/slice.rb index cacb59cca..ff59cff2f 100644 --- a/lib/ontologies_linked_data/models/slice.rb +++ b/lib/ontologies_linked_data/models/slice.rb @@ -40,18 +40,25 @@ def self.synchronize_groups_to_slices # Check to make sure each group has a corresponding slice (and ontologies match) groups = LinkedData::Models::Group.where.include(LinkedData::Models::Group.attributes(:all)).all groups.each do |g| - slice = self.find(g.acronym).include(LinkedData::Models::Slice.attributes(:all)).first + acronym = g.acronym.to_s.downcase.gsub(" ", "-") + ontologies = g.ontologies || [] + slice = self.find(acronym).include(LinkedData::Models::Slice.attributes(:all)).first + + next if ontologies.empty? && slice.nil? + if slice - slice.ontologies = g.ontologies + next if ontologies.empty? + + slice.ontologies = ontologies slice.save if slice.valid? else slice = self.new({ - acronym: g.acronym.downcase.gsub(" ", "-"), + acronym: acronym, name: g.name, description: g.description, - ontologies: g.ontologies + ontologies: ontologies }) - slice.save + slice.save if slice.valid? end end end diff --git a/lib/ontologies_linked_data/models/submission_status.rb b/lib/ontologies_linked_data/models/submission_status.rb index 29ca397f2..1048ffcc8 100644 --- a/lib/ontologies_linked_data/models/submission_status.rb +++ b/lib/ontologies_linked_data/models/submission_status.rb @@ -7,6 +7,7 @@ class SubmissionStatus < LinkedData::Models::Base "RDF_LABELS", "ERROR_RDF_LABELS", "OBSOLETE", "ERROR_OBSOLETE", "INDEXED", "ERROR_INDEXED", + "INDEXED_ALL_DATA", "ERROR_INDEXED_ALL_DATA", "INDEXED_PROPERTIES", "ERROR_INDEXED_PROPERTIES", "METRICS", "ERROR_METRICS", "ANNOTATOR", "ERROR_ANNOTATOR", @@ -19,6 +20,8 @@ class SubmissionStatus < LinkedData::Models::Base "RDF_ERROR" => "Error parsing", "INDEXED" => "Indexed terms for search", "ERROR_INDEXED" => "Error indexing terms for search", + "INDEXED_ALL_DATA" => "Indexed all the data of the resource", + "ERROR_INDEXED_ALL_DATA" => "Error indexeding all the data of the resource", "INDEXED_PROPERTIES" => "Indexed properties for search", "ERROR_INDEXED_PROPERTIES" => "Error indexing properties for search", "METRICS" => "Class metrics calculated", @@ -87,7 +90,7 @@ def self.readable_statuses(statuses) begin readable_stats = statuses_raw.map { |s| s.bring(:code); s.code } - rescue Exception => e + rescue Exception readable_stats = nil if i == num_calls diff --git a/lib/ontologies_linked_data/models/users/oauth_authentication.rb b/lib/ontologies_linked_data/models/users/oauth_authentication.rb new file mode 100644 index 000000000..01e52adb4 --- /dev/null +++ b/lib/ontologies_linked_data/models/users/oauth_authentication.rb @@ -0,0 +1,186 @@ +require 'bcrypt' +require 'openssl' +require 'base64' +require 'json' +require 'jwt' +require 'faraday' +require 'active_support/core_ext/object/blank' + +module LinkedData + module Models + module Users + module OAuthAuthentication + + def self.included base + base.extend ClassMethods + end + + module ClassMethods + + def oauth_providers + LinkedData.settings.oauth_providers + end + + def oauth_authenticate(token, provider) + user_data = case provider.to_sym + when :github + auth_github(token) + when :google + auth_google(token) + when :orcid + auth_orcid(token) + when :keycloak + auth_keycloak(token) + else + nil + end + + create_if_not_exists(user_data) if user_data + end + + private + + def create_if_not_exists(user_data) + user = user_by_email(user_data[:email]) + + if user.nil? + auth_create_user(user_data) + else + sync_providers_id(user, user_data[:githubId], user_data[:orcidId]) + end + end + + def sync_providers_id(user, github_id, orcid_id) + user.bring_remaining + user.githubId = github_id if user.githubId.blank? && !github_id.blank? + user.orcidId = orcid_id if user.orcidId.blank? && !orcid_id.blank? + user.save(override_security: true) if user.valid? + user + end + + def auth_create_user(user_data) + user = User.new(user_data) + user.password = SecureRandom.hex(16) + + return nil unless user.valid? + + user.save(send_notifications: true) + user + end + + def user_by_email(email) + LinkedData::Models::User.where(email: email).first unless email.nil? + end + + def user_from_orcid_data(user_data) + { + email: user_data['email'], + firstName: user_data['name']['given-names'], + lastName: user_data['name']['family-name'], + username: user_data['email'].split('@').first, + orcidId: user_data['orcid'] + } + end + + def auth_orcid(token) + user_data = token_check(token, :orcid) + + return nil if user_data.nil? + + user_from_orcid_data user_data + + end + + def user_from_google_data(user_data) + { + email: user_data['email'], + firstName: user_data['given_name'], + lastName: user_data['family_name'], + username: user_data['email'].split('@').first + } + end + + def auth_google(token) + user_data = token_check(token, :google) + + return nil if user_data.nil? + + user_from_google_data user_data + end + + def auth_github(token) + user_data = token_check(token, :github) + + return nil if user_data.nil? + + user_from_github_data user_data + end + + def user_from_github_data(user_data) + { + email: user_data['email'], + username: user_data['login'], + firstName: user_data['name'].split(' ').first, + lastName: user_data['name'].split(' ').drop(1).join(' '), + githubId: user_data['login'] + } + end + + def user_from_keycloak_data(user_data) + { + email: user_data['email'], + username: user_data['preferred_username'], + firstName: user_data['given_name'], + lastName: user_data['family_name'] + } + end + + def auth_keycloak(token) + user_data = token_check(token, :keycloak) + + return nil if user_data.nil? + + user_from_keycloak_data user_data + end + + def token_check(token, provider) + provider_config = oauth_providers[provider.to_sym] + + return nil unless provider_config + + if provider_config[:check].eql?(:access_token) + access_token_check(token, provider_config[:link]) + elsif provider_config[:check].eql?(:jwt_token) + jwt_token_check(token, provider_config[:cert]) + end + end + + def jwt_token_check(jwt_token, cert) + decode_cert = Base64.decode64(cert) + rsa_public = OpenSSL::X509::Certificate.new(decode_cert).public_key + begin + JWT.decode(jwt_token, rsa_public, true, { algorithm: 'HS256' }) + rescue JWT::DecodeError + nil + end + end + + def access_token_check(token, link) + response = Faraday.new(url: link) do |faraday| + faraday.headers['Authorization'] = "Bearer #{token}" + faraday.adapter Faraday.default_adapter + end.get + + return nil unless response.success? + + JSON.parse(response.body) + end + end + + end + + end + + end +end + diff --git a/lib/ontologies_linked_data/models/users/user.rb b/lib/ontologies_linked_data/models/users/user.rb index e27cfbd1a..7f2a96fb8 100644 --- a/lib/ontologies_linked_data/models/users/user.rb +++ b/lib/ontologies_linked_data/models/users/user.rb @@ -3,12 +3,18 @@ require 'ontologies_linked_data/models/users/authentication' require 'ontologies_linked_data/models/users/role' require 'ontologies_linked_data/models/users/subscription' +require 'ontologies_linked_data/models/users/oauth_authentication' module LinkedData module Models class User < LinkedData::Models::Base include BCrypt include LinkedData::Models::Users::Authentication + include LinkedData::Models::Users::OAuthAuthentication + include LinkedData::Concerns::Analytics + + ANALYTICS_REDIS_FIELD = "user_analytics" + PAGES_ANALYTICS_REDIS_FIELD = "pages_analytics" attr_accessor :show_apikey @@ -18,9 +24,11 @@ class User < LinkedData::Models::Base attribute :role, enforce: [:role, :list], :default => lambda {|x| [LinkedData::Models::Users::Role.default]} attribute :firstName, enforce: [:safe_text_128] attribute :lastName, enforce: [:safe_text_128] + attribute :subscribed, default: false attribute :githubId, enforce: [:unique] attribute :orcidId, enforce: [:unique] attribute :created, enforce: [:date_time], :default => lambda { |record| DateTime.now } + attribute :lastLoginAt, enforce: [:date_time], :default => lambda { |record| DateTime.now } attribute :passwordHash, enforce: [:existence] attribute :apikey, enforce: [:unique], :default => lambda {|x| SecureRandom.uuid} attribute :subscription, enforce: [:list, :subscription] @@ -28,16 +36,19 @@ class User < LinkedData::Models::Base attribute :resetToken attribute :resetTokenExpireTime attribute :provisionalClasses, inverse: { on: :provisional_class, attribute: :creator } + attribute :createdOntologies, enforce: [:list], handler: :load_created_ontologies # Hypermedia settings embed :subscription embed_values :role => [:role] serialize_default :username, :email, :role, :apikey serialize_never :passwordHash, :show_apikey, :resetToken, :resetTokenExpireTime - serialize_filter lambda {|inst| show_apikey?(inst)} + serialize_filter lambda {|inst| filter_attributes(inst)} system_controlled :created, :resetToken, :resetTokenExpireTime + link_to LinkedData::Hypermedia::Link.new("createdOntologies", lambda {|s| "users/#{s.id.split('/').last}/ontologies"}, nil) + # Cache cache_timeout 3600 @@ -54,6 +65,23 @@ def self.show_apikey?(inst) end end + def self.show_lastLoginAt?(attrs) + unless Thread.current[:remote_user]&.admin? + return attrs - [:lastLoginAt] + end + return attrs + end + + def self.filter_attributes(inst) + attrs = show_apikey?(inst) + attrs = show_lastLoginAt?(attrs) + attrs + end + + def embedded_doc + self.to_s + end + def initialize(attributes = {}) # Don't allow passwordHash to be set here attributes.delete(:passwordHash) @@ -69,13 +97,44 @@ def initialize(attributes = {}) self end + def update_last_login + self.lastLoginAt = DateTime.now + self.save(override_security: true) + end + def save(*args) # Reset ontology cache if user changes their custom set if LinkedData.settings.enable_http_cache && self.modified_attributes.include?(:customOntology) Ontology.cache_collection_invalidate OntologySubmission.cache_collection_invalidate end + super + + if args.first&.dig(:send_notifications) + begin + LinkedData::Utils::Notifications.new_user(self) + rescue StandardError => e + puts "Error on user creation notification: #{e.message}" + end + end + self + end + + def load_created_ontologies + ontologies = [] + q = Goo.sparql_query_client.select(:id, :acronym, :administeredBy).distinct + .from(Ontology.uri_type) + .where( + [:id, LinkedData::Models::Ontology.attribute_uri(:administeredBy), :administeredBy], + [:id, LinkedData::Models::Ontology.attribute_uri(:acronym), :acronym], + ) + .filter("?administeredBy = <#{self.id}>") + acronyms = q.execute.map { |o| o.acronym.to_s } + return ontologies if acronyms.empty? + filter_by_acronym = Goo::Filter.new(:acronym).regex("^(#{acronyms.join('|')})$") + ontologies = Ontology.where.include(Ontology.goo_attrs_to_load([:all])).filter(filter_by_acronym).all + return ontologies end def admin? @@ -96,12 +155,20 @@ def custom_ontology_id_set def to_s if self.bring?(:username) - LinkedData::Utils::Triples.last_iri_fragment self.id.to_s + LinkedData::Utils::Triples.last_iri_fragment(self.id.to_s) else self.username.to_s end end + def self.analytics_redis_key + ANALYTICS_REDIS_FIELD + end + + def self.page_visits_analytics + load_data(PAGES_ANALYTICS_REDIS_FIELD) + end + private def set_passwordHash(password) diff --git a/lib/ontologies_linked_data/monkeypatches/rdf_raptor_ffi.rb b/lib/ontologies_linked_data/monkeypatches/rdf_raptor_ffi.rb new file mode 100644 index 000000000..9b728d918 --- /dev/null +++ b/lib/ontologies_linked_data/monkeypatches/rdf_raptor_ffi.rb @@ -0,0 +1,22 @@ +require 'rdf/raptor' + +module LinkedData + module Monkeypatches + module RdfRaptorFfi + module_function + + # rdf-raptor 3.1/3.2 binds raptor_free_world with zero args, but calls + # it with a pointer from the AutoPointer finalizer. Linux + ffi 1.17.x + # raises ArgumentError on shutdown unless the signature is corrected. + def apply! + return unless defined?(RDF::Raptor::FFI::V2) + + v2 = RDF::Raptor::FFI::V2 + v2.singleton_class.send(:remove_method, :raptor_free_world) if v2.respond_to?(:raptor_free_world) + v2.attach_function :raptor_free_world, [:pointer], :void + end + end + end +end + +LinkedData::Monkeypatches::RdfRaptorFfi.apply! diff --git a/lib/ontologies_linked_data/parser/owlapi.rb b/lib/ontologies_linked_data/parser/owlapi.rb index ab3016674..0a1540c91 100644 --- a/lib/ontologies_linked_data/parser/owlapi.rb +++ b/lib/ontologies_linked_data/parser/owlapi.rb @@ -124,7 +124,7 @@ def call_owlapi_java_command end if not File.exist?(File.join([@output_repo, "owlapi.xrdf"])) raise Parser::OWLAPIParserException, "OWLAPI java command exited with"+ - " #{w.value.exitstatus}. " +\ + " #{w.value.exitstatus}. " + \ "Output file #{File.join([@output_repo, "owlapi.xrdf"])} cannot be found." else @file_triples_path = File.join([@output_repo, "owlapi.xrdf"]) diff --git a/lib/ontologies_linked_data/purl/purl_client.rb b/lib/ontologies_linked_data/purl/purl_client.rb index 6180d07b7..eaa560bf4 100644 --- a/lib/ontologies_linked_data/purl/purl_client.rb +++ b/lib/ontologies_linked_data/purl/purl_client.rb @@ -63,7 +63,7 @@ def purl_exists(acronym) def delete_purl(acronym) headers = purl_server_login() http = get_http() - res, data = http.delete(PURL_ADMIN_PATH.call(acronym), headers) + res, _ = http.delete(PURL_ADMIN_PATH.call(acronym), headers) return res.code == "200" end @@ -78,4 +78,4 @@ def get_http() return http end end -end \ No newline at end of file +end diff --git a/lib/ontologies_linked_data/sample_data/ontology.rb b/lib/ontologies_linked_data/sample_data/ontology.rb index 0ed9ac085..82760cffc 100644 --- a/lib/ontologies_linked_data/sample_data/ontology.rb +++ b/lib/ontologies_linked_data/sample_data/ontology.rb @@ -25,7 +25,7 @@ def self.create_ontologies_and_submissions(options = {}) acronym = options[:acronym] || "TEST-ONT" pref_label_property = options[:pref_label_property] || false synonym_property = options[:synonym_property] || false - definition_property = options[:synonym_property] || false + definition_property = options[:definition_property] || false name = options[:name] # set ontology type ontology_type = nil @@ -72,7 +72,10 @@ def self.create_ontologies_and_submissions(options = {}) submissionId: o.next_submission_id, definitionProperty: (RDF::IRI.new "http://bioontology.org/ontologies/biositemap.owl#definition"), contact: [contact], - released: DateTime.now - 3 + released: DateTime.now - 3, + uri: RDF::URI.new("https://test-#{o.next_submission_id}.com"), + description: "Description #{o.next_submission_id}", + status: 'production' } sub_options[:prefLabelProperty] = RDF::IRI.new(pref_label_property) if pref_label_property sub_options[:synonymProperty] = RDF::IRI.new(synonym_property) if synonym_property @@ -135,7 +138,7 @@ def self.load_semantic_types_ontology(options = {}) file_path = options[:file_path] file_path = "../../../../test/data/ontology_files/umls_semantictypes.ttl" if file_path.nil? - count, acronyms, sty = create_ontologies_and_submissions({ + _, _, sty = create_ontologies_and_submissions({ ont_count: 1, submission_count: 1, process_submission: true, @@ -179,7 +182,7 @@ def self.delete_ontologies_and_submissions def self.sample_owl_ontologies(process_submission: false, process_options: nil) process_options ||= {process_rdf: true, extract_metadata: false, index_search: false} - count, acronyms, bro = create_ontologies_and_submissions({ + _, _, bro = create_ontologies_and_submissions({ process_submission: process_submission, process_options: process_options, acronym: "BROTEST", @@ -190,7 +193,7 @@ def self.sample_owl_ontologies(process_submission: false, process_options: nil) }) # This one has some nasty looking IRIS with slashes in the anchor - count, acronyms, mccl = create_ontologies_and_submissions({ + _, _, mccl = create_ontologies_and_submissions({ process_submission: process_submission, process_options: process_options, acronym: "MCCLTEST", @@ -201,7 +204,7 @@ def self.sample_owl_ontologies(process_submission: false, process_options: nil) }) # This one has resources wih accents. - count, acronyms, onto_matest = create_ontologies_and_submissions({ + _, _, onto_matest = create_ontologies_and_submissions({ process_submission: process_submission, process_options: process_options, acronym: "ONTOMATEST", diff --git a/lib/ontologies_linked_data/security/authorization.rb b/lib/ontologies_linked_data/security/authorization.rb index 7e2463268..c552b9337 100644 --- a/lib/ontologies_linked_data/security/authorization.rb +++ b/lib/ontologies_linked_data/security/authorization.rb @@ -4,46 +4,52 @@ module LinkedData module Security class Authorization APIKEYS_FOR_AUTHORIZATION = {} + USER_APIKEY_PARAM = 'userapikey'.freeze + API_KEY_PARAM = 'apikey'.freeze def initialize(app = nil) @app = app end ROUTES_THAT_BYPASS_SECURITY = Set.new([ - "/", - "/documentation", - "/jsonview/jsonview.css", - "/jsonview/jsonview.js" - ]) + "/", + "/documentation", + "/jsonview/jsonview.css", + "/jsonview/jsonview.js" + ]) def call(env) req = Rack::Request.new(env) params = req.params + apikey = find_apikey(env, params) + status = 200 + error_message = '' - unless apikey + if !apikey status = 401 - response = { - status: status, - error: "You must provide an API Key either using the query-string parameter `apikey` or the `Authorization` header: `Authorization: apikey token=my_apikey`. " + \ - "Your API Key can be obtained by logging in at #{LinkedData.settings.ui_host}/account" - } - end - - if status != 401 && !authorized?(apikey, env) + error_message = <<-MESSAGE + You must provide an API Key either using the query-string parameter `apikey` or the `Authorization` header: `Authorization: apikey token=my_apikey`. + Your API Key can be obtained by logging in at #{LinkedData.settings.ui_host}/account" + MESSAGE + elsif !authorized?(apikey, env) status = 401 - response = { - status: status, - error: "You must provide a valid API Key. " + \ - "Your API Key can be obtained by logging in at #{LinkedData.settings.ui_host}/account" - } + error_message = "You must provide a valid API Key. Your API Key can be obtained by logging in at #{LinkedData.settings.ui_host}/account" end - if status == 401 && !bypass?(env) + response = { + status: status, + error: error_message + } + + if status.eql?(401) && !bypass?(env) LinkedData::Serializer.build_response(env, status: status, body: response) else + # unfrozen params so that they can be encoded by Rack using occurring after updating the gem RDF to v3.0 + env["rack.request.form_hash"]&.transform_values!(&:dup) + env["rack.request.query_hash"]&.transform_values!(&:dup) status, headers, response = @app.call(env) - apikey_cookie(env, headers, apikey, params) + save_apikey_in_cookie(env, headers, apikey, params) [status, headers, response] end end @@ -57,57 +63,56 @@ def bypass?(env) ## # Inject a cookie with the API Key if it is present and we're in HTML content type - def apikey_cookie(env, headers, apikey, params) + COOKIE_APIKEY_PARAM = "ncbo_apikey" + + def save_apikey_in_cookie(env, headers, apikey, params) # If we're using HTML, inject the apikey in a cookie (ignores bad accept headers) + best = nil begin best = LinkedData::Serializer.best_response_type(env, params) - rescue LinkedData::Serializer::AcceptHeaderError; end - if best == LinkedData::MediaTypes::HTML - Rack::Utils.set_cookie_header!(headers, "ncbo_apikey", {:value => apikey, :path => "/", :expires => Time.now+90*24*60*60}) + rescue LinkedData::Serializer::AcceptHeaderError + # Ignored end + + return unless best == LinkedData::MediaTypes::HTML + + Rack::Utils.set_cookie_header!(headers, COOKIE_APIKEY_PARAM, { + value: apikey, + path: '/', + expires: Time.now + 90 * 24 * 60 * 60 + }) end def find_apikey(env, params) - apikey = nil - header_auth = env["HTTP_AUTHORIZATION"] || env["Authorization"] - if params["apikey"] && params["userapikey"] - apikey_authed = authorized?(params["apikey"], env) - return unless apikey_authed - apikey = params["userapikey"] - elsif params["apikey"] - apikey = params["apikey"] - elsif apikey.nil? && header_auth - token = Rack::Utils.parse_query(header_auth.split(" ")[1]) - return unless token["token"] - - # Strip spaces from start and end of string - apikey = token["token"].gsub(/\"/, "") - # If the user apikey is passed, use that instead - if token["userapikey"] && !token["userapikey"].empty? - apikey_authed = authorized?(apikey, env) - return unless apikey_authed - apikey = token["userapikey"].gsub(/\"/, "") - end - elsif apikey.nil? && env["HTTP_COOKIE"] && env["HTTP_COOKIE"].include?("ncbo_apikey") - cookie = Rack::Utils.parse_query(env["HTTP_COOKIE"]) - apikey = cookie["ncbo_apikey"] if cookie["ncbo_apikey"] - end - apikey + apikey = user_apikey(env, params) + return apikey if apikey + + apikey = params[API_KEY_PARAM] + return apikey if apikey + + apikey = request_header_apikey(env) + return apikey if apikey + + cookie_apikey(env) end def authorized?(apikey, env) return false if apikey.nil? + if APIKEYS_FOR_AUTHORIZATION.key?(apikey) store_user(APIKEYS_FOR_AUTHORIZATION[apikey], env) else - users = LinkedData::Models::User.where(apikey: apikey).include(LinkedData::Models::User.attributes(:all)).to_a - return false if users.empty? + user = LinkedData::Models::User.where(apikey: apikey) + .include(LinkedData::Models::User.attributes(:all)) + .first + return false if user.nil? + # This will kind-of break if multiple apikeys exist # Though it is also kind-of ok since we just want to know if a user with corresponding key exists - user = users.first + store_user(user, env) end - return true + true end def store_user(user, env) @@ -115,6 +120,51 @@ def store_user(user, env) env.update("REMOTE_USER" => user) end + private + + def request_header_apikey(env) + header_auth = get_header_auth(env) + return if header_auth.empty? + + auth_type, token_string = header_auth.split(/\s+/, 2) + return unless auth_type&.downcase == API_KEY_PARAM + return if token_string.nil? || token_string.empty? + + token = Rack::Utils.parse_query(token_string) + return if token['token'].nil? || token['token'].empty? + # Strip spaces from start and end of string + apikey = token['token'].gsub(/\"/, "") + # If the user apikey is passed, use that instead + if token[USER_APIKEY_PARAM] && !token[USER_APIKEY_PARAM].empty? + apikey_authed = authorized?(apikey, env) + return unless apikey_authed + + apikey = token[USER_APIKEY_PARAM].gsub(/\"/, "") + end + apikey + end + + def cookie_apikey(env) + return unless env["HTTP_COOKIE"] + + cookie = Rack::Utils.parse_cookies(env) + cookie[COOKIE_APIKEY_PARAM] + end + + def get_header_auth(env) + env["HTTP_AUTHORIZATION"] || env["Authorization"] || '' + end + + def user_apikey(env, params) + return unless (params["apikey"] && params["userapikey"]) + + apikey_authed = authorized?(params[API_KEY_PARAM], env) + + return unless apikey_authed + + params[USER_APIKEY_PARAM] + end + end end end diff --git a/lib/ontologies_linked_data/serializers/json.rb b/lib/ontologies_linked_data/serializers/json.rb index ce77050ac..187b460d7 100644 --- a/lib/ontologies_linked_data/serializers/json.rb +++ b/lib/ontologies_linked_data/serializers/json.rb @@ -7,78 +7,109 @@ class JSON def self.serialize(obj, options = {}) + return serialize_mod_objects(obj, options) if mod_object?(obj) + # Handle the serialization for all other objects in the standard way hash = obj.to_flex_hash(options) do |hash, hashed_obj| - current_cls = hashed_obj.respond_to?(:klass) ? hashed_obj.klass : hashed_obj.class - result_lang = self.get_languages(get_object_submission(hashed_obj), options[:lang]) + process_common_serialization(hash, hashed_obj, options) + end + MultiJson.dump(hash) + end + + def self.serialize_mod_objects(obj, options = {}) + # using one context and links in mod objects + global_context = {} - # Add the id to json-ld attribute - if current_cls.ancestors.include?(LinkedData::Hypermedia::Resource) && !current_cls.embedded? && hashed_obj.respond_to?(:id) - prefixed_id = LinkedData::Models::Base.replace_url_id_to_prefix(hashed_obj.id) - hash["@id"] = prefixed_id.to_s - end + hash = obj.to_flex_hash(options) do |hash, hashed_obj| + process_common_serialization(hash, hashed_obj, options, global_context) + end - # Add the type - hash["@type"] = type(current_cls, hashed_obj) if hash["@id"] - - # Generate links - # NOTE: If this logic changes, also change in xml.rb - if generate_links?(options) - links = LinkedData::Hypermedia.generate_links(hashed_obj) - unless links.empty? - hash["links"] = links - hash["links"].merge!(generate_links_context(hashed_obj)) if generate_context?(options) - end - end + result = {} + # handle adding the context for HydraPage + if obj.is_a?(LinkedData::Models::HydraPage) + global_context["@context"] ||= {} + global_context["@context"].merge!(LinkedData::Models::HydraPage.generate_hydra_context) + end + result.merge!(global_context) unless global_context.empty? + result.merge!(hash) if hash.is_a?(Hash) + MultiJson.dump(result) + end + + + private + + def self.process_common_serialization(hash, hashed_obj, options, global_context= nil) + current_cls = hashed_obj.respond_to?(:klass) ? hashed_obj.klass : hashed_obj.class + result_lang = get_languages(get_object_submission(hashed_obj), options[:lang]) + + add_id_and_type(hash, hashed_obj, current_cls) + add_links(hash, hashed_obj, options) if generate_links?(options) + add_context(hash, hashed_obj, options, current_cls, result_lang, global_context) if generate_context?(options) + end - # Generate context + def self.add_id_and_type(hash, hashed_obj, current_cls) + return unless current_cls.ancestors.include?(LinkedData::Hypermedia::Resource) && !current_cls.embedded? && hashed_obj.respond_to?(:id) + + hash["@id"] = LinkedData::Models::Base.replace_url_id_to_prefix(hashed_obj.id).to_s + hash["@type"] = type(current_cls, hashed_obj) if hash["@id"] + end + + def self.add_links(hash, hashed_obj, options) + links = LinkedData::Hypermedia.generate_links(hashed_obj) + return if links.empty? + hash["links"] = links + hash["links"].merge!(generate_links_context(hashed_obj)) if generate_context?(options) + end + + def self.add_context(hash, hashed_obj, options, current_cls, result_lang, global_context) + return if global_context&.any? + + if global_context.nil? if current_cls.ancestors.include?(Goo::Base::Resource) && !current_cls.embedded? - if generate_context?(options) - context = generate_context(hashed_obj, hash.keys, options) - hash.merge!(context) - end + context = generate_context(hashed_obj, hash.keys, options) + hash.merge!(context) + elsif (hashed_obj.instance_of?(LinkedData::Models::ExternalClass) || hashed_obj.instance_of?(LinkedData::Models::InterportalClass)) && !current_cls.embedded? + # Add context for ExternalClass + external_class_context = { "@context" => { "@vocab" => Goo.vocabulary.to_s, "prefLabel" => "http://data.bioontology.org/metadata/skosprefLabel" } } + hash.merge!(external_class_context) end hash['@context']['@language'] = result_lang if hash['@context'] + elsif global_context.empty? + context = generate_context(hashed_obj, hash.keys, options) + global_context.replace(context) + global_context["@context"]["@language"] = result_lang unless global_context.empty? end - MultiJson.dump(hash) end - private + def self.mod_object?(obj) + return false if obj.nil? + single_object = (obj.class == Array) && obj.any? ? obj.first : obj + single_object.class.ancestors.include?(LinkedData::Models::HydraPage) || single_object.class.ancestors.include?(LinkedData::Models::ModBase) + end + def self.get_object_submission(obj) obj.class.respond_to?(:attributes) && obj.class.attributes.include?(:submission) ? obj.submission : nil end def self.get_languages(submission, user_languages) - - if user_languages.eql?(:all) || user_languages.blank? - if submission - submission.bring(:naturalLanguage) if submission.bring?(:naturalLanguage) - submission_languages_languages = get_submission_languages(submission.naturalLanguage) - end - - if submission_languages_languages.blank? - result_lang = [Goo.main_languages.first.to_s] - else - result_lang = [submission_languages_languages.first] - end - else - result_lang = Array(user_languages) + result_lang = user_languages + + if submission + submission.bring :naturalLanguage + languages = get_submission_languages(submission.naturalLanguage) + # intersection of the two arrays , if the requested language is not :all + result_lang = user_languages == :all ? languages : Array(user_languages) & languages + result_lang = result_lang.first if result_lang.length == 1 end - if result_lang.length == 1 - result_lang.first - elsif result_lang.empty? - Goo.main_languages.first.to_s - else - result_lang - end + result_lang end def self.get_submission_languages(submission_natural_language = []) submission_natural_language = submission_natural_language.values.flatten if submission_natural_language.is_a?(Hash) - submission_natural_language.map { |natural_language| natural_language.to_s.split('/').last[0..1].to_sym }.compact - end + submission_natural_language.map { |natural_language| natural_language.to_s['iso639'] && natural_language.to_s.split('/').last[0..1].to_sym }.compact + end def self.type(current_cls, hashed_obj) if current_cls.respond_to?(:type_uri) @@ -113,12 +144,7 @@ def self.generate_context(object, serialized_attrs = [], options = {}) predicate = { "@id" => linked_model.type_uri.to_s, "@type" => "@id" } else # use the original predicate property if set - predicate_attr = if current_cls.model_settings[:attributes][attr][:property].is_a?(Proc) - attr - else - current_cls.model_settings[:attributes][attr][:property] || attr - end - + predicate_attr = current_cls.model_settings[:attributes][attr][:property] || attr # predicate with custom namespace # if the namespace can be resolved by the namespaces added in Goo then it will be resolved. predicate = "#{Goo.vocabulary(current_cls.model_settings[:attributes][attr][:namespace])&.to_s}#{predicate_attr}" diff --git a/lib/ontologies_linked_data/serializers/jsonld.rb b/lib/ontologies_linked_data/serializers/jsonld.rb new file mode 100644 index 000000000..22e6b7d6d --- /dev/null +++ b/lib/ontologies_linked_data/serializers/jsonld.rb @@ -0,0 +1,40 @@ +require 'multi_json' +require 'json/ld' + +module LinkedData + module Serializers + class JSONLD + + def self.serialize(hashes, options = {}) + subject = RDF::URI.new(hashes['id']) + reverse = hashes['reverse'] || {} + hashes.delete('id') + hashes.delete('reverse') + graph = RDF::Graph.new + + hashes.each do |property_url, val| + Array(val).each do |v| + if v.is_a?(Hash) + blank_node = RDF::Node.new + v.each do |blank_predicate, blank_value| + graph << RDF::Statement.new(blank_node, RDF::URI.new(blank_predicate), blank_value) + end + v = blank_node + end + graph << RDF::Statement.new(subject, RDF::URI.new(property_url), v) + end + end + + reverse.each do |reverse_subject, reverse_property| + Array(reverse_property).each do |s| + graph << RDF::Statement.new(RDF::URI.new(reverse_subject), RDF::URI.new(s), subject) + end + end + + context = { '@context' => options.transform_keys(&:to_s) } + compacted = ::JSON::LD::API.compact(::JSON::LD::API.fromRdf(graph), context['@context']) + MultiJson.dump(compacted) + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/serializers/jsonp.rb b/lib/ontologies_linked_data/serializers/jsonp.rb index 48a28ccf6..58cb5e1cf 100644 --- a/lib/ontologies_linked_data/serializers/jsonp.rb +++ b/lib/ontologies_linked_data/serializers/jsonp.rb @@ -5,7 +5,7 @@ def self.serialize(obj, options) callback = options[:params]["callback"] || options[:params]["jsonp"] || "?" variable = options[:params]["variable"] json = LinkedData::Serializers::JSON.serialize(obj, options) - response = begin + begin if callback && variable "var #{variable} = #{json};\n#{callback}(#{variable});" elsif variable @@ -19,4 +19,4 @@ def self.serialize(obj, options) end end end -end \ No newline at end of file +end diff --git a/lib/ontologies_linked_data/serializers/ntriples.rb b/lib/ontologies_linked_data/serializers/ntriples.rb new file mode 100644 index 000000000..c96795a77 --- /dev/null +++ b/lib/ontologies_linked_data/serializers/ntriples.rb @@ -0,0 +1,37 @@ +module LinkedData + module Serializers + class NTRIPLES + + def self.serialize(hashes, options = {}) + subject = RDF::URI.new(hashes['id']) + reverse = hashes['reverse'] || {} + hashes.delete('id') + hashes.delete('reverse') + RDF::Writer.for(:ntriples).buffer(prefixes: options) do |writer| + hashes.each do |p, o| + predicate = RDF::URI.new(p) + Array(o).each do |item| + if item.is_a?(Hash) + blank_node = RDF::Node.new + item.each do |blank_predicate, blank_value| + writer << RDF::Statement.new(blank_node, RDF::URI.new(blank_predicate), blank_value) + end + item = blank_node + end + writer << RDF::Statement.new(subject, predicate, item) + end + end + + reverse.each do |reverse_subject, reverse_property| + Array(reverse_property).each do |s| + writer << RDF::Statement.new(RDF::URI.new(reverse_subject), RDF::URI.new(s), subject) + end + end + end + end + + end + end +end + + \ No newline at end of file diff --git a/lib/ontologies_linked_data/serializers/rdf_xml.rb b/lib/ontologies_linked_data/serializers/rdf_xml.rb new file mode 100644 index 000000000..e06590f00 --- /dev/null +++ b/lib/ontologies_linked_data/serializers/rdf_xml.rb @@ -0,0 +1,43 @@ +module LinkedData + module Serializers + class RDF_XML + def self.serialize(hashes, options = {}) + subject = RDF::URI.new(hashes["id"]) + reverse = hashes["reverse"] || {} + hashes.delete("id") + hashes.delete("reverse") + graph = RDF::Graph.new + + hashes.each do |property_url, val| + Array(val).each do |v| + if v.is_a?(Hash) + blank_node = RDF::Node.new + v.each do |blank_predicate, blank_value| + graph << RDF::Statement.new(blank_node, RDF::URI.new(blank_predicate), blank_value) + end + v = blank_node + end + graph << RDF::Statement.new(subject, RDF::URI.new(property_url), v) + end + end + + inverse_graph = RDF::Graph.new + reverse.each do |reverse_subject, reverse_property| + Array(reverse_property).each do |s| + inverse_graph << RDF::Statement.new(RDF::URI.new(reverse_subject), RDF::URI.new(s), subject) + end + end + + a = RDF::RDFXML::Writer.buffer(prefixes: options) do |writer| + writer << graph + end + + b = RDF::RDFXML::Writer.buffer(prefixes: options) do |writer| + writer << inverse_graph + end + xml_result = "#{a.chomp("\n")}\n#{b.sub!(/^<\?xml[^>]*>\n]*>/, '').gsub(/^$\n/, '')}" + xml_result.gsub(/^$\n/, '') + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/serializers/serializers.rb b/lib/ontologies_linked_data/serializers/serializers.rb index b6006280d..1603c1dbe 100644 --- a/lib/ontologies_linked_data/serializers/serializers.rb +++ b/lib/ontologies_linked_data/serializers/serializers.rb @@ -1,8 +1,12 @@ require 'ontologies_linked_data/media_types' require 'ontologies_linked_data/serializers/xml' +require 'ontologies_linked_data/serializers/rdf_xml' require 'ontologies_linked_data/serializers/json' require 'ontologies_linked_data/serializers/jsonp' +require 'ontologies_linked_data/serializers/jsonld' require 'ontologies_linked_data/serializers/html' +require 'ontologies_linked_data/serializers/ntriples' +require 'ontologies_linked_data/serializers/turtle' module LinkedData module Serializers @@ -10,17 +14,15 @@ def self.serialize(obj, type, options = {}) SERIALIZERS[type].serialize(obj, options) end - class Turtle - def self.serialize(obj, options) - end - end - SERIALIZERS = { LinkedData::MediaTypes::HTML => HTML, LinkedData::MediaTypes::JSON => JSON, LinkedData::MediaTypes::JSONP => JSONP, + LinkedData::MediaTypes::JSONLD => JSONLD, LinkedData::MediaTypes::XML => XML, - LinkedData::MediaTypes::TURTLE => JSON + LinkedData::MediaTypes::RDF_XML => RDF_XML, + LinkedData::MediaTypes::TURTLE => TURTLE, + LinkedData::MediaTypes::NTRIPLES => NTRIPLES } end end \ No newline at end of file diff --git a/lib/ontologies_linked_data/serializers/turtle.rb b/lib/ontologies_linked_data/serializers/turtle.rb new file mode 100644 index 000000000..b0cc9ecf4 --- /dev/null +++ b/lib/ontologies_linked_data/serializers/turtle.rb @@ -0,0 +1,38 @@ +module LinkedData + module Serializers + class TURTLE + def self.serialize(hashes, options = {}) + subject = RDF::URI.new(hashes['id']) + reverse = hashes['reverse'] || {} + hashes.delete('id') + hashes.delete('reverse') + options.delete(:rdf) + + RDF::Writer.for(:turtle).buffer(prefixes: options) do |writer| + hashes.each do |p, o| + predicate = RDF::URI.new(p) + Array(o).each do |item| + if item.is_a?(Hash) + blank_node = RDF::Node.new + item.each do |blank_predicate, blank_value| + writer << RDF::Statement.new(blank_node, RDF::URI.new(blank_predicate), blank_value) + end + item = blank_node + end + writer << RDF::Statement.new(subject, predicate, item) + end + end + + reverse.each do |reverse_subject, reverse_property| + Array(reverse_property).each do |s| + writer << RDF::Statement.new(RDF::URI.new(reverse_subject), RDF::URI.new(s), subject) + end + end + + end + end + end + end +end + + \ No newline at end of file diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_all_data_indexer.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_all_data_indexer.rb new file mode 100644 index 000000000..1b278a70b --- /dev/null +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_all_data_indexer.rb @@ -0,0 +1,167 @@ +require 'parallel' +module LinkedData + module Services + class OntologySubmissionAllDataIndexer < OntologySubmissionProcess + + def process(logger, options = nil) + status = LinkedData::Models::SubmissionStatus.find('INDEXED_ALL_DATA').first + begin + index_all_data(logger, **options) + @submission.add_submission_status(status) + rescue StandardError => e + logger.error("Error indexing all data for submission: #{e.message} : #{e.backtrace.join("\n")}") + @submission.add_submission_status(status.get_error_status) + ensure + @submission.save + end + end + + private + + def index_all_data(logger, commit: true) + size = Goo.backend_vo? ? 100 : 1000 + count_ids = 0 + @submission.bring_remaining + @submission.ontology.bring_remaining + ontology_acronym = @submission.ontology.acronym + + conn = init_search_collection(ontology_acronym) + r = Goo.sparql_query_client.query("SELECT (COUNT(DISTINCT ?id) as ?count) WHERE { GRAPH <#{@submission.id}> { ?id ?p ?v } }") + total_ids = r.each_solution.first[:count].to_i + logger.info "Total ids count: #{total_ids}" + + r = Goo.sparql_query_client.query("SELECT (COUNT(*) as ?count) WHERE { GRAPH <#{@submission.id}> { ?id ?p ?v } }") + total_triples = r.each_solution.first[:count].to_i + logger.info "Total triples count: #{total_triples}" + + chunk_size = total_ids / size + 1 + total_triples_indexed = 0 + total_time = Benchmark.realtime do + results = Parallel.map((1..chunk_size).to_a, in_threads: 10) do |p| + index_all_data_page(logger, p, size, ontology_acronym, conn, commit) + end + results.each do |x| + next if x.nil? + + count_ids += x[1] + total_triples_indexed += x[2] + end + end + + logger.info("Completed indexing all ontology data in #{total_time} sec. (#{count_ids} ids / #{total_triples} triples)") + end + + def index_all_data_page(logger, page, size, ontology_acronym, conn, commit = true) + ids = [] + time = Benchmark.realtime do + ids = fetch_ids(size, page) + end + count_ids = ids.size + total_time = time + return if ids.empty? + + logger.info("Page #{page} - Fetch IDS: #{ids.size} ids (total: #{count_ids}) in #{time} sec.") + documents = [] + triples_count = 0 + time = Benchmark.realtime do + documents, triples_count = fetch_triples(ids, ontology_acronym) + end + total_time += time + logger.info("Page #{page} - Fetch IDs triples: #{triples_count} in #{time} sec.") + return if documents.empty? + + time = Benchmark.realtime do + puts "Indexing #{documents.size} documents page: #{page}" + conn.index_document(documents.values, commit: commit) + end + logger.info("Page #{page} - Indexed #{documents.size} documents page: #{page} in #{time} sec.") + [total_time, count_ids, triples_count] + end + + def fetch_ids(size, page) + query = Goo.sparql_query_client.select(:id) + .distinct + .from(RDF::URI.new(@submission.id)) + .where(%i[id p v]) + .limit(size) + .offset((page - 1) * size) + + query.each_solution.map{|x| x.id.to_s} + end + + def update_doc(doc, property, new_val) + unescaped_prop = property.gsub('___', '://') + + unescaped_prop = unescaped_prop.gsub('_', '/') + existent_val = doc["#{unescaped_prop}_t"] || doc["#{unescaped_prop}_txt"] + + if !existent_val && !property['#'] + unescaped_prop = unescaped_prop.sub(%r{/([^/]+)$}, '#\1') # change latest '/' with '#' + existent_val = doc["#{unescaped_prop}_t"] || doc["#{unescaped_prop}_txt"] + end + + if existent_val && new_val || new_val.is_a?(Array) + doc.delete("#{unescaped_prop}_t") + doc["#{unescaped_prop}_txt"] = Array(existent_val) + Array(new_val).map(&:to_s) + elsif existent_val.nil? && new_val + doc["#{unescaped_prop}_t"] = new_val.to_s + end + doc + end + + def init_search_collection(ontology_acronym) + @submission.class.clear_indexed_content(ontology_acronym) + end + + def fetch_triples(ids_slice, ontology_acronym) + documents = {} + count = 0 + fetch_paginated_triples(ids_slice).each do |sol| + count += 1 + doc = documents[sol[:id].to_s] + doc ||= { + id: "#{sol[:id]}_#{ontology_acronym}", submission_id_t: @submission.id.to_s, + ontology_t: ontology_acronym, resource_model: @submission.class.model_name, + resource_id: sol[:id].to_s + } + property = sol[:p].to_s + value = sol[:v] + + if property.to_s.eql?(RDF.type.to_s) + update_doc(doc, 'type', value) + else + update_doc(doc, property, value) + end + documents[sol[:id].to_s] = doc + end + + [documents, count] + end + + def fetch_paginated_triples(ids_slice) + solutions = [] + count = 0 + page = 1 + page_size = 10_000 + filter = ids_slice.map { |x| "?id = <#{x}>" }.join(' || ') + + while count.positive? || page == 1 + query = Goo.sparql_query_client.select(:id, :p, :v) + .from(RDF::URI.new(@submission.id)) + .where(%i[id p v]) + .filter(filter) + .slice((page - 1) * page_size, page_size) + + sol = query.each_solution.to_a + count = sol.size + solutions += sol + break if count.zero? || count < page_size + + page += 1 + end + solutions + end + end + end + +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb index f5e18e346..cde750f58 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_archiver.rb @@ -6,21 +6,23 @@ class OntologySubmissionArchiver < OntologySubmissionProcess FOLDERS_TO_DELETE = ['unzipped'] FILE_SIZE_ZIPPING_THRESHOLD = 100 * 1024 * 1024 # 100MB - def process - archive_submission + def process(force: false) + archive_submission(force: force) end private - def archive_submission + def archive_submission(force: false) @submission.ontology.bring(:submissions) submissions = @submission.ontology.submissions return if submissions.nil? submissions.each { |s| s.bring(:submissionId) } submission = submissions.sort { |a, b| b.submissionId <=> a.submissionId }.first + latest_submission_id = submission&.submissionId || 0 + @submission.bring(:submissionId) if @submission.bring?(:submissionId) - return unless @submission.submissionId < submission.submissionId + return if (@submission.submissionId >= latest_submission_id) && !force @submission.submissionStatus = nil status = LinkedData::Models::SubmissionStatus.find("ARCHIVED").first @@ -29,6 +31,7 @@ def archive_submission @submission.unindex # Delete everything except for original ontology file. + delete_old_graph delete_old_submission_files @submission.uploadFilePath = zip_submission_uploaded_file end @@ -57,7 +60,11 @@ def delete_old_submission_files submission_folders.each { |d| FileUtils.remove_dir(d) if File.directory?(d) } end + def delete_old_graph + Goo.sparql_data_client.delete_graph(@submission.id) + end + end end -end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_extract_metadata.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_extract_metadata.rb new file mode 100644 index 000000000..c7c575908 --- /dev/null +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_extract_metadata.rb @@ -0,0 +1,325 @@ +module LinkedData + module Services + class SubmissionMetadataExtractor < OntologySubmissionProcess + + def process(logger, options = nil) + extract_metadata(logger, options[:user_params], heavy_extraction: options[:heavy_extraction].eql?(true)) + end + + private + + def extract_metadata(logger = nil, user_params, heavy_extraction: true) + logger ||= Logger.new(STDOUT) + logger.info("Extracting metadata from the ontology submission #{@submission.id}") + + unless @submission.valid? + logger.error("Cannot extract metadata from #{@submission.id} because the submission is invalid") + return + end + + ontology_iri = extract_ontology_iri + + if ontology_iri + old_uri = @submission.uri + @submission.uri = ontology_iri + + if @submission.valid? + @submission.save + else + logger.error("The extracted submission URI #{ontology_iri} is invalid. Discarding...") + @submission.uri = old_uri + end + end + + if heavy_extraction + begin + # Extract metadata directly from the ontology + extract_ontology_metadata(logger, user_params, skip_attrs: [:uri]) + logger.info('Additional metadata extracted') + rescue StandardError => e + message = e.message + "\n\n " + e.backtrace.join("\n ") + logger.error("Error while extracting additional metadata: #{message}") + end + end + + version_exists = @submission.version + + if version_exists.to_s.strip.empty? + version_info = extract_version + + if version_info + @submission.version = version_info + + unless @submission.valid? + logger.error("The extracted submission version \"#{version_info}\" is invalid. Discarding...") + @submission.version = nil + end + end + end + + @submission.save + end + + def extract_version + query = Goo.sparql_query_client.select(:versionInfo).distinct + .from(@submission.id) + .where([RDF::URI.new('http://bioportal.bioontology.org/ontologies/versionSubject'), + RDF::URI.new("#{Goo.namespaces[:owl].to_s}versionInfo"), + :versionInfo]) + sol = query.each_solution.first || {} + sol[:versionInfo]&.to_s + end + + def extract_ontology_iri + query = Goo.sparql_query_client.select(:uri).distinct + .from(@submission.id) + .where([:uri, + RDF::URI.new("#{Goo.namespaces[:rdf].to_s}type"), + RDF::URI.new("#{Goo.namespaces[:owl].to_s}Ontology")]) + sol = query.each_solution.first || {} + RDF::URI.new(sol[:uri]) if sol[:uri] + end + + # Extract additional metadata about the ontology + # First it extracts the main metadata, then the mapped metadata + def extract_ontology_metadata(logger, user_params, skip_attrs: []) + # The commented out line below would give priority to the existing values over extracted values + # user_params = object_to_hash(@submission) unless user_params + user_params = {} if user_params.nil? || !user_params + ontology_uri = @submission.uri + logger.info("Extracting additional metadata from ontology #{ontology_uri}") + + # go through all OntologySubmission attributes. Returns symbols + LinkedData::Models::OntologySubmission.attributes(:all).each do |attr| + next if skip_attrs.include? attr + # for attribute with the :extractedMetadata setting on, and that have not been defined by the user + attr_settings = LinkedData::Models::OntologySubmission.attribute_settings(attr) + + # Check if the user provided a non-empty value for this attribute + user_provided_value = user_params.key?(attr) && !empty_value?(user_params[attr]) + # Proceed only if the attribute is marked as extractable and the user did NOT provide a value + should_extract = attr_settings[:extractedMetadata] && !user_provided_value + next unless should_extract + + # a boolean to check if a value that should be single have already been extracted + single_extracted = false + type = enforce?(attr, :list) ? :list : :string + old_value = value(attr, type) + + unless attr_settings[:namespace].nil? + property_to_extract = "#{attr_settings[:namespace].to_s}:#{attr.to_s}" + hash_results = extract_each_metadata(ontology_uri, attr, property_to_extract, logger) + single_extracted = send_value(attr, hash_results, logger) unless hash_results.empty? + end + + # extracts attribute value from metadata mappings + attr_settings[:metadataMappings] ||= [] + + attr_settings[:metadataMappings].each do |mapping| + break if single_extracted + + hash_mapping_results = extract_each_metadata(ontology_uri, attr, mapping.to_s, logger) + single_extracted = send_value(attr, hash_mapping_results, logger) unless hash_mapping_results.empty? + end + + new_value = value(attr, type) + + if empty_value?(new_value) && !empty_value?(old_value) + logger.error("Extracted attribute #{attr} with the value #{new_value} is empty. Returning to the old value: #{old_value}.") + send_value(attr, old_value, logger) + end + end + end + + def object_to_hash(obj) + obj.instance_variables.each_with_object({}) do |ivar, hash| + key = ivar.to_s.delete("@").to_sym + value = obj.instance_variable_get(ivar) + hash[key] = value + end + end + + def empty_value?(value) + value.nil? || (value.is_a?(Array) && value.empty?) || value.to_s.strip.empty? + end + + def value(attr, type) + val = @submission.send(attr.to_s) + type.eql?(:list) ? Array(val) || [] : val || '' + end + + def send_value(attr, new_value, logger) + old_val = nil + single_extracted = false + + if enforce?(attr, :list) + old_val = value(attr, :list) + old_values = old_val.dup + new_values = new_value.values + new_values = new_values.map { |v| find_or_create_agent(attr, v, logger) }.compact if enforce?(attr, :Agent) + + old_values.push(*new_values) + + @submission.send("#{attr}=", old_values.uniq) + elsif enforce?(attr, :concatenate) + # if multiple value for this attribute, then we concatenate it + # Add the concat at the very end, to easily join the content of the array + old_val = value(attr, :string) + metadata_values = old_val.split(', ') + new_values = new_value.values.map { |x| x.to_s.split(', ') }.flatten + + @submission.send("#{attr}=", (metadata_values + new_values).uniq.join(', ')) + else + new_value = new_value.is_a?(Hash)? new_value.values.first : new_value.to_s + new_value = find_or_create_agent(attr, nil, logger) if enforce?(attr, :Agent) + @submission.send("#{attr}=", new_value) + old_val = @submission.previous_values&.key?(attr) ? @submission.previous_values[attr] : nil + single_extracted = true + end + + unless @submission.valid? + logger.error("Error while extracting metadata for the attribute #{attr}: #{@submission.errors[attr] || @submission.errors}") + new_value&.delete if enforce?(attr, :Agent) && new_value.respond_to?(:delete) + @submission.send("#{attr}=", old_val) + end + + single_extracted + end + + # Return a hash with the best literal value for an URI + # it selects the literal according to their language: no language > english > french > other languages + def select_metadata_literal(metadata_uri, metadata_literal, hash) + return unless metadata_literal.is_a?(RDF::Literal) + + if hash.key?(metadata_uri) + if metadata_literal.has_language? + if !hash[metadata_uri].has_language? + return hash + else + case metadata_literal.language + when :en, :eng + # Take the value with english language over other languages + hash[metadata_uri] = metadata_literal.to_s + return hash + when :fr, :fre + # If no english, take french + if hash[metadata_uri].language == :en || hash[metadata_uri].language == :eng + return hash + else + hash[metadata_uri] = metadata_literal.to_s + return hash + end + else + return hash + end + end + else + # Take the value with no language in priority (considered as a default) + hash[metadata_uri] = metadata_literal.to_s + return hash + end + else + hash[metadata_uri] = metadata_literal.to_s + hash + end + end + + # A function to extract additional metadata + # Take the literal data if the property is pointing to a literal + # If pointing to an URI: first it takes the "omv:name" of the object pointed by the property, if nil it takes the "rdfs:label". + # If not found it check for "omv:firstName + omv:lastName" (for "omv:Person") of this object. And to finish it takes the "URI" + # The hash_results contains the metadataUri (objet pointed on by the metadata property) with the value we are using from it + def extract_each_metadata(ontology_uri, attr, prop_to_extract, logger) + + query_metadata = < #{prop_to_extract} ?extractedObject . +OPTIONAL { ?extractedObject omv:name ?omvname } . +OPTIONAL { ?extractedObject omv:firstName ?omvfirstname } . +OPTIONAL { ?extractedObject omv:lastName ?omvlastname } . +OPTIONAL { ?extractedObject rdfs:label ?rdfslabel } . +} +eos + Goo.namespaces.each do |prefix, uri| + query_metadata = "PREFIX #{prefix}: <#{uri}>\n" + query_metadata + end + + # logger.info(query_metadata) + # This hash will contain the "literal" metadata for each object (uri or literal) pointed by the metadata predicate + hash_results = {} + Goo.sparql_query_client.query(query_metadata).each_solution do |sol| + value = sol[:extractedObject] + if enforce?(attr, :uri) + # If the attr is enforced as URI then it directly takes the URI + uri_value = value ? RDF::URI.new(value.to_s.strip) : nil + hash_results[value] = uri_value if uri_value&.valid? + elsif enforce?(attr, :date_time) + begin + hash_results[value] = DateTime.iso8601(value.to_s) + rescue StandardError => e + logger.error("Impossible to extract DateTime metadata for #{attr}: #{value}. It should follow iso8601 standards. Error message: #{e}") + end + elsif enforce?(attr, :integer) + begin + hash_results[value] = value.to_s.to_i + rescue StandardError => e + logger.error("Impossible to extract integer metadata for #{attr}: #{value}. Error message: #{e}") + end + elsif enforce?(attr, :boolean) + case value.to_s.downcase + when 'true' + hash_results[value] = true + when 'false' + hash_results[value] = false + else + logger.error("Impossible to extract boolean metadata for #{attr}: #{value}. Error message: #{e}") + end + elsif value.is_a?(RDF::URI) + hash_results = find_object_label(hash_results, sol, value) + else + # If this is directly a literal + hash_results = select_metadata_literal(value, value, hash_results) + end + end + hash_results + end + + def find_object_label(hash_results, sol, value) + if !sol[:omvname].nil? + hash_results = select_metadata_literal(value, sol[:omvname], hash_results) + elsif !sol[:rdfslabel].nil? + hash_results = select_metadata_literal(value, sol[:rdfslabel], hash_results) + elsif !sol[:omvfirstname].nil? + hash_results = select_metadata_literal(value, sol[:omvfirstname], hash_results) + # if first and last name are defined (for omv:Person) + hash_results[value] = "#{hash_results[value]} #{sol[:omvlastname]}" unless sol[:omvlastname].nil? + elsif !sol[:omvlastname].nil? + # if only last name is defined + hash_results = select_metadata_literal(value, sol[:omvlastname], hash_results) + else + # if the object is an URI but we are requesting a String + hash_results[value] = value.to_s + end + hash_results + end + + def enforce?(attr, type) + LinkedData::Models::OntologySubmission.attribute_settings(attr)[:enforce].include?(type) + end + + def find_or_create_agent(attr, old_val, logger) + agent = LinkedData::Models::Agent.where(agentType: 'person', name: old_val).first + begin + agent ||= LinkedData::Models::Agent.new(name: old_val, agentType: 'person', creator: @submission.ontology.administeredBy.first).save + rescue + logger.error("Error while extracting metadata for the attribute #{attr}: Can't create Agent #{agent.errors} ") + agent = nil + end + agent + end + end + end +end diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_indexer.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_indexer.rb index 7aec2b385..9a50b7f76 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_indexer.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_indexer.rb @@ -26,7 +26,7 @@ def process_indexation(logger, options) def index(logger, commit = true, optimize = true) page = 0 - size = 1000 + size = 2500 count_classes = 0 time = Benchmark.realtime do @@ -118,7 +118,7 @@ def index(logger, commit = true, optimize = true) begin # this cal is needed for indexing of properties LinkedData::Models::Class.map_attributes(c, paging.equivalent_predicates, include_languages: true) - rescue Exception => e + rescue Exception i = 0 num_calls = LinkedData.settings.num_retries_4store success = nil diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb new file mode 100644 index 000000000..176d2a7bf --- /dev/null +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb @@ -0,0 +1,320 @@ +module LinkedData + module Services + + class GenerateMissingLabels < OntologySubmissionProcess + + def process(logger, options = {}) + handle_missing_labels(options[:file_path], logger) + end + + private + + def handle_missing_labels(file_path, logger) + callbacks = { + include_languages: true, + missing_labels: { + op_name: 'Missing Labels Generation', + required: true, + status: LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first, + artifacts: { + file_path: file_path + }, + caller_on_pre: :generate_missing_labels_pre, + caller_on_pre_page: :generate_missing_labels_pre_page, + caller_on_each: :generate_missing_labels_each, + caller_on_post_page: :generate_missing_labels_post_page, + caller_on_post: :generate_missing_labels_post + } + } + + raw_paging = LinkedData::Models::Class.in(@submission).include(:prefLabel, :synonym, :label) + loop_classes(logger, raw_paging, @submission, callbacks) + end + + def process_callbacks(logger, callbacks, action_name) + callbacks.delete_if do |_, callback| + begin + if callback[action_name] + callable = self.method(callback[action_name]) + yield(callable, callback) + end + false + rescue Exception => e + logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") + logger.flush + + if callback[:status] + @submission.add_submission_status(callback[:status].get_error_status) + @submission.save + end + + # halt the entire processing if :required is set to true + raise e if callback[:required] + # continue processing of other callbacks, but not this one + true + end + end + end + + def loop_classes(logger, raw_paging, submission, callbacks) + page = 1 + size = 2500 + count_classes = 0 + acr = submission.id.to_s.split("/")[-1] + + # include all languages in attributes of classes if asked for + incl_lang = callbacks.delete(:include_languages) + RequestStore.store[:requested_lang] = :ALL if incl_lang + operations = callbacks.values.map { |v| v[:op_name] }.join(", ") + + time = Benchmark.realtime do + paging = raw_paging.page(page, size) + cls_count_set = false + cls_count = submission.class_count(logger) + + if cls_count > -1 + # prevent a COUNT SPARQL query if possible + paging.page_count_set(cls_count) + cls_count_set = true + else + cls_count = 0 + end + + iterate_classes = false + # 1. init artifacts hash if not explicitly passed in the callback + # 2. determine if class-level iteration is required + callbacks.each { |_, callback| callback[:artifacts] ||= {}; + if callback[:caller_on_each] + iterate_classes = true + end } + + process_callbacks(logger, callbacks, :caller_on_pre) { + |callable, callback| callable.call(callback[:artifacts], logger, paging) } + + page_len = -1 + prev_page_len = -1 + + begin + t0 = Time.now + page_classes = paging.page(page, size).all + total_pages = page_classes.total_pages + page_len = page_classes.length + + # nothing retrieved even though we're expecting more records + if total_pages > 0 && page_classes.empty? && (prev_page_len == -1 || prev_page_len == size) + j = 0 + num_calls = LinkedData.settings.num_retries_4store + + while page_classes.empty? && j < num_calls do + j += 1 + logger.error("Empty page encountered. Retrying #{j} times...") + sleep(2) + page_classes = paging.page(page, size).all + unless page_classes.empty? + logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...") + end + end + + if page_classes.empty? + msg = "Empty page #{page} of #{total_pages} persisted after retrying #{j} times. #{operations} of #{acr} aborted..." + logger.error(msg) + raise msg + end + end + + if page_classes.empty? + if total_pages > 0 + logger.info("The number of pages reported for #{acr} - #{total_pages} is higher than expected #{page - 1}. Completing #{operations}...") + else + logger.info("Ontology #{acr} contains #{total_pages} pages...") + end + break + end + + prev_page_len = page_len + logger.info("#{acr}: page #{page} of #{total_pages} - #{page_len} ontology terms retrieved in #{Time.now - t0} sec.") + logger.flush + count_classes += page_classes.length + + process_callbacks(logger, callbacks, :caller_on_pre_page) { + |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page) } + + page_classes.each { |c| + process_callbacks(logger, callbacks, :caller_on_each) { + |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page, c) } + } if iterate_classes + + process_callbacks(logger, callbacks, :caller_on_post_page) { + |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page) } + cls_count += page_classes.length unless cls_count_set + + page = page_classes.next? ? page + 1 : nil + end while !page.nil? + + callbacks.each { |_, callback| callback[:artifacts][:count_classes] = cls_count } + process_callbacks(logger, callbacks, :caller_on_post) { + |callable, callback| callable.call(callback[:artifacts], logger, paging) } + end + + logger.info("Completed #{operations}: #{acr} in #{time} sec. #{count_classes} classes.") + logger.flush + + # set the status on actions that have completed successfully + callbacks.each do |_, callback| + if callback[:status] + @submission.add_submission_status(callback[:status]) + @submission.save + end + end + RequestStore.store[:requested_lang] = nil if incl_lang + end + + def generate_missing_labels_pre(artifacts = {}, logger, paging) + file_path = artifacts[:file_path] + artifacts[:save_in_file] = File.join(File.dirname(file_path), "labels.ttl") + artifacts[:save_in_file_mappings] = File.join(File.dirname(file_path), "mappings.ttl") + property_triples = LinkedData::Utils::Triples.rdf_for_custom_properties(@submission) + Goo.sparql_data_client.append_triples(@submission.id, property_triples, mime_type = "application/x-turtle") + fsave = File.open(artifacts[:save_in_file], "w") + fsave.write(property_triples) + fsave_mappings = File.open(artifacts[:save_in_file_mappings], "w") + artifacts[:fsave] = fsave + artifacts[:fsave_mappings] = fsave_mappings + end + + def generate_missing_labels_pre_page(artifacts = {}, logger, paging, page_classes, page) + artifacts[:label_triples] = [] + artifacts[:mapping_triples] = [] + end + + def generate_missing_labels_each(artifacts = {}, logger, paging, page_classes, page, c) + prefLabel = nil + portal_lang = Goo.portal_language + prefLabel_lang = c.prefLabel(include_languages: true) + no_default_prefLabel = prefLabel_lang.nil? || (prefLabel_lang.keys & [portal_lang, :none, '@none']).empty? + + if prefLabel_lang.nil? || no_default_prefLabel + lang_rdfs_labels = c.label(include_languages: true) + + if lang_rdfs_labels.is_a?(Array) + lang_rdfs_labels = lang_rdfs_labels.empty? ? { none: [] } : { none: lang_rdfs_labels } + elsif lang_rdfs_labels.to_a.empty? || + (no_default_prefLabel && lang_rdfs_labels.is_a?(Hash) && (lang_rdfs_labels.keys & [portal_lang, :none, '@none']).empty?) + lang_rdfs_labels = lang_rdfs_labels.is_a?(Hash) ? lang_rdfs_labels : {} + lang_rdfs_labels[:none] = [] + end + + # if portal language label exists but no generic prefLabel is defined, copy the portal language + # label into the generic one (lang_rdfs_labels[:none]). Otherwise, this case results in nil + # generic prefLabel for a class, even though prefLabels exist for multiple languages. + # For Example (portal_lang = :en): + # + # + # As defined in http://en.wikipedia.org/wiki/Gene_therapy + # Gene Therapy + # Thérapie génique + # + lang_rdfs_labels[:none] = lang_rdfs_labels[portal_lang].dup if lang_rdfs_labels[:none].blank? && + lang_rdfs_labels.key?(portal_lang) + + # prefLabel (if defined) takes priority over label + prefLabel_lang ||= {} + prefLabel_lang.each do |lang, value| + next if value.nil? || (value.respond_to?(:empty?) && value.empty?) + + lang = :none if lang == :"@none" + lang_rdfs_labels[lang] = Array(value) + end + + lang_rdfs_labels.each do |lang, rdfs_labels| + synonyms = c.synonym + synonyms_present = !(synonyms.nil? || (synonyms.respond_to?(:empty?) && synonyms.empty?)) + + if rdfs_labels && rdfs_labels.length > 1 && synonyms_present + rdfs_labels = (Set.new(c.label) - Set.new(c.synonym)).to_a.first + rdfs_labels = c.label if rdfs_labels.nil? || rdfs_labels.length == 0 + end + + rdfs_labels = [rdfs_labels] if rdfs_labels && !rdfs_labels.instance_of?(Array) + + label = nil + if rdfs_labels && rdfs_labels.length > 0 + # this sort is needed for a predictable label selection + label = rdfs_labels.sort[0] + else + label = LinkedData::Utils::Triples.last_iri_fragment c.id.to_s + end + + # Set language to nil for :none and assign pref_label + lang = nil if lang.eql?(:none) || lang.to_s.eql?('@none') + prefLabel = label + + artifacts[:label_triples] << LinkedData::Utils::Triples.label_for_class_triple( + c.id, Goo.vocabulary(:metadata_def)[:prefLabel], prefLabel, lang) + end + elsif prefLabel_lang + prefLabel = c.prefLabel + else + prefLabel = LinkedData::Utils::Triples.last_iri_fragment c.id.to_s + end + + if @submission.ontology.viewOf.nil? + loomLabel = LinkedData::Models::OntologySubmission.loom_transform_literal(prefLabel.to_s) + + if loomLabel.length > 2 + artifacts[:mapping_triples] << LinkedData::Utils::Triples.loom_mapping_triple( + c.id, Goo.vocabulary(:metadata_def)[:mappingLoom], loomLabel) + end + artifacts[:mapping_triples] << LinkedData::Utils::Triples.uri_mapping_triple( + c.id, Goo.vocabulary(:metadata_def)[:mappingSameURI], c.id) + end + end + + def generate_missing_labels_post_page(artifacts = {}, logger, paging, page_classes, page) + rest_mappings = LinkedData::Mappings.migrate_rest_mappings(@submission.ontology.acronym) + artifacts[:mapping_triples].concat(rest_mappings) + + if artifacts[:label_triples].length > 0 + logger.info("Asserting #{artifacts[:label_triples].length} labels in " + + "#{@submission.id.to_ntriples}") + logger.flush + artifacts[:label_triples] = artifacts[:label_triples].join("\n") + artifacts[:fsave].write(artifacts[:label_triples]) + t0 = Time.now + Goo.sparql_data_client.append_triples(@submission.id, artifacts[:label_triples], mime_type = "application/x-turtle") + t1 = Time.now + logger.info("Labels asserted in #{t1 - t0} sec.") + logger.flush + else + logger.info("No labels generated in page #{page}.") + logger.flush + end + + if artifacts[:mapping_triples].length > 0 + logger.info("Asserting #{artifacts[:mapping_triples].length} mappings in " + + "#{@submission.id.to_ntriples}") + logger.flush + artifacts[:mapping_triples] = artifacts[:mapping_triples].join("\n") + artifacts[:fsave_mappings].write(artifacts[:mapping_triples]) + + t0 = Time.now + Goo.sparql_data_client.append_triples(@submission.id, artifacts[:mapping_triples], mime_type = "application/x-turtle") + t1 = Time.now + logger.info("Mapping labels asserted in #{t1 - t0} sec.") + logger.flush + end + end + + def generate_missing_labels_post(artifacts = {}, logger, pagging) + logger.info("end generate_missing_labels traversed #{artifacts[:count_classes]} classes") + logger.info("Saved generated labels in #{artifacts[:save_in_file]}") + artifacts[:fsave].close() + artifacts[:fsave_mappings].close() + logger.flush + end + + end + + end +end + diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_obsolete_classes.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_obsolete_classes.rb new file mode 100644 index 000000000..5487ba6af --- /dev/null +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_obsolete_classes.rb @@ -0,0 +1,81 @@ +module LinkedData + module Services + + class ObsoleteClassesGenerator < OntologySubmissionProcess + + def process(logger, options) + status = LinkedData::Models::SubmissionStatus.find('OBSOLETE').first + begin + generate_obsolete_classes(logger, options[:file_path]) + @submission.add_submission_status(status) + @submission.save + rescue Exception => e + logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") + logger.flush + @submission.add_submission_status(status.get_error_status) + @submission.save + # if obsolete fails the parsing fails + raise e + end + @submission + end + + private + + def generate_obsolete_classes(logger, file_path) + @submission.bring(:obsoleteProperty) if @submission.bring?(:obsoleteProperty) + @submission.bring(:obsoleteParent) if @submission.bring?(:obsoleteParent) + classes_deprecated = [] + if @submission.obsoleteProperty && + @submission.obsoleteProperty.to_s != "http://www.w3.org/2002/07/owl#deprecated" + + predicate_obsolete = RDF::URI.new(@submission.obsoleteProperty.to_s) + query_obsolete_predicate = < 0 + classes_deprecated.uniq! + logger.info("Asserting owl:deprecated statement for #{classes_deprecated} classes") + save_in_file = File.join(File.dirname(file_path), "obsolete.ttl") + fsave = File.open(save_in_file, "w") + classes_deprecated.each do |class_id| + fsave.write(LinkedData::Utils::Triples.obselete_class_triple(class_id) + "\n") + end + fsave.close() + Goo.sparql_data_client.append_triples_from_file( + @submission.id, + save_in_file, + mime_type = "application/x-turtle") + end + end + end + end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_properties_indexer.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_properties_indexer.rb index beefd048a..612723763 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_properties_indexer.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_properties_indexer.rb @@ -42,14 +42,14 @@ def index_properties(logger, commit: true, optimize: true) props.each_slice(size) do |prop_batch| t = Time.now - LinkedData::Models::Class.indexBatch(prop_batch, :property) + LinkedData::Models::OntologyProperty.indexBatch(prop_batch) logger.info("Page #{page} of ontology properties indexed in #{Time.now - t} seconds."); logger.flush page += 1 end if commit t0 = Time.now - LinkedData::Models::Class.indexCommit(nil, :property) + LinkedData::Models::OntologyProperty.indexCommit logger.info("Ontology properties index commit in #{Time.now - t0} seconds.") end end @@ -59,7 +59,7 @@ def index_properties(logger, commit: true, optimize: true) if optimize logger.info('Optimizing ontology properties index...') time = Benchmark.realtime do - LinkedData::Models::Class.indexOptimize(nil, :property) + LinkedData::Models::OntologyProperty.indexOptimize end logger.info("Completed optimizing ontology properties index in #{time} seconds.") end diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_rdf_generator.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_rdf_generator.rb index de4cec06d..25bec2615 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_rdf_generator.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_rdf_generator.rb @@ -1,293 +1,6 @@ module LinkedData module Services - class MissingLabelsHandler < OntologySubmissionProcess - - def process(logger, options = {}) - handle_missing_labels(options[:file_path], logger) - end - - private - - def handle_missing_labels(file_path, logger) - callbacks = { - include_languages: true, - missing_labels: { - op_name: 'Missing Labels Generation', - required: true, - status: LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first, - artifacts: { - file_path: file_path - }, - caller_on_pre: :generate_missing_labels_pre, - caller_on_pre_page: :generate_missing_labels_pre_page, - caller_on_each: :generate_missing_labels_each, - caller_on_post_page: :generate_missing_labels_post_page, - caller_on_post: :generate_missing_labels_post - } - } - - raw_paging = LinkedData::Models::Class.in(@submission).include(:prefLabel, :synonym, :label) - loop_classes(logger, raw_paging, @submission, callbacks) - end - - def process_callbacks(logger, callbacks, action_name) - callbacks.delete_if do |_, callback| - begin - if callback[action_name] - callable = self.method(callback[action_name]) - yield(callable, callback) - end - false - rescue Exception => e - logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") - logger.flush - - if callback[:status] - @submission.add_submission_status(callback[:status].get_error_status) - @submission.save - end - - # halt the entire processing if :required is set to true - raise e if callback[:required] - # continue processing of other callbacks, but not this one - true - end - end - end - - def loop_classes(logger, raw_paging, submission, callbacks) - page = 1 - size = 2500 - count_classes = 0 - acr = submission.id.to_s.split("/")[-1] - - # include all languages in attributes of classes if asked for - incl_lang = callbacks.delete(:include_languages) - RequestStore.store[:requested_lang] = :ALL if incl_lang - operations = callbacks.values.map { |v| v[:op_name] }.join(", ") - - time = Benchmark.realtime do - paging = raw_paging.page(page, size) - cls_count_set = false - cls_count = submission.class_count(logger) - - if cls_count > -1 - # prevent a COUNT SPARQL query if possible - paging.page_count_set(cls_count) - cls_count_set = true - else - cls_count = 0 - end - - iterate_classes = false - # 1. init artifacts hash if not explicitly passed in the callback - # 2. determine if class-level iteration is required - callbacks.each { |_, callback| callback[:artifacts] ||= {}; - if callback[:caller_on_each] - iterate_classes = true - end } - - process_callbacks(logger, callbacks, :caller_on_pre) { - |callable, callback| callable.call(callback[:artifacts], logger, paging) } - - page_len = -1 - prev_page_len = -1 - - begin - t0 = Time.now - page_classes = paging.page(page, size).all - total_pages = page_classes.total_pages - page_len = page_classes.length - - # nothing retrieved even though we're expecting more records - if total_pages > 0 && page_classes.empty? && (prev_page_len == -1 || prev_page_len == size) - j = 0 - num_calls = LinkedData.settings.num_retries_4store - - while page_classes.empty? && j < num_calls do - j += 1 - logger.error("Empty page encountered. Retrying #{j} times...") - sleep(2) - page_classes = paging.page(page, size).all - unless page_classes.empty? - logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...") - end - end - - if page_classes.empty? - msg = "Empty page #{page} of #{total_pages} persisted after retrying #{j} times. #{operations} of #{acr} aborted..." - logger.error(msg) - raise msg - end - end - - if page_classes.empty? - if total_pages > 0 - logger.info("The number of pages reported for #{acr} - #{total_pages} is higher than expected #{page - 1}. Completing #{operations}...") - else - logger.info("Ontology #{acr} contains #{total_pages} pages...") - end - break - end - - prev_page_len = page_len - logger.info("#{acr}: page #{page} of #{total_pages} - #{page_len} ontology terms retrieved in #{Time.now - t0} sec.") - logger.flush - count_classes += page_classes.length - - process_callbacks(logger, callbacks, :caller_on_pre_page) { - |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page) } - - page_classes.each { |c| - process_callbacks(logger, callbacks, :caller_on_each) { - |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page, c) } - } if iterate_classes - - process_callbacks(logger, callbacks, :caller_on_post_page) { - |callable, callback| callable.call(callback[:artifacts], logger, paging, page_classes, page) } - cls_count += page_classes.length unless cls_count_set - - page = page_classes.next? ? page + 1 : nil - end while !page.nil? - - callbacks.each { |_, callback| callback[:artifacts][:count_classes] = cls_count } - process_callbacks(logger, callbacks, :caller_on_post) { - |callable, callback| callable.call(callback[:artifacts], logger, paging) } - end - - logger.info("Completed #{operations}: #{acr} in #{time} sec. #{count_classes} classes.") - logger.flush - - # set the status on actions that have completed successfully - callbacks.each do |_, callback| - if callback[:status] - @submission.add_submission_status(callback[:status]) - @submission.save - end - end - RequestStore.store[:requested_lang] = nil if incl_lang - end - - def generate_missing_labels_pre(artifacts = {}, logger, paging) - file_path = artifacts[:file_path] - artifacts[:save_in_file] = File.join(File.dirname(file_path), "labels.ttl") - artifacts[:save_in_file_mappings] = File.join(File.dirname(file_path), "mappings.ttl") - property_triples = LinkedData::Utils::Triples.rdf_for_custom_properties(@submission) - Goo.sparql_data_client.append_triples(@submission.id, property_triples, mime_type = "application/x-turtle") - fsave = File.open(artifacts[:save_in_file], "w") - fsave.write(property_triples) - fsave_mappings = File.open(artifacts[:save_in_file_mappings], "w") - artifacts[:fsave] = fsave - artifacts[:fsave_mappings] = fsave_mappings - end - - def generate_missing_labels_pre_page(artifacts = {}, logger, paging, page_classes, page) - artifacts[:label_triples] = [] - artifacts[:mapping_triples] = [] - end - - def generate_missing_labels_each(artifacts = {}, logger, paging, page_classes, page, c) - prefLabel = nil - portal_lang = Goo.portal_language - prefLabel_lang = c.prefLabel(include_languages: true) - no_default_prefLabel = prefLabel_lang.nil? || (prefLabel_lang.keys & [portal_lang, :none]).empty? - - if prefLabel_lang.nil? || no_default_prefLabel - lang_rdfs_labels = c.label(include_languages: true) - - if lang_rdfs_labels.to_a.empty? || - lang_rdfs_labels.is_a?(Array) || - (no_default_prefLabel && lang_rdfs_labels.is_a?(Hash) && (lang_rdfs_labels.keys & [portal_lang, :none]).empty?) - lang_rdfs_labels = lang_rdfs_labels.is_a?(Hash) ? lang_rdfs_labels : {} - lang_rdfs_labels[:none] = [] - end - - lang_rdfs_labels.each do |lang, rdfs_labels| - if rdfs_labels && rdfs_labels.length > 1 && c.synonym.length > 0 - rdfs_labels = (Set.new(c.label) - Set.new(c.synonym)).to_a.first - rdfs_labels = c.label if rdfs_labels.nil? || rdfs_labels.length == 0 - end - - rdfs_labels = [rdfs_labels] if rdfs_labels and not (rdfs_labels.instance_of? Array) - - label = nil - if rdfs_labels && rdfs_labels.length > 0 - # this sort is needed for a predictable label selection - label = rdfs_labels.sort[0] - else - label = LinkedData::Utils::Triples.last_iri_fragment c.id.to_s - end - - lang = nil if lang === :none - prefLabel = label - - artifacts[:label_triples] << LinkedData::Utils::Triples.label_for_class_triple( - c.id, Goo.vocabulary(:metadata_def)[:prefLabel], prefLabel, lang) - end - elsif prefLabel_lang - prefLabel = c.prefLabel - else - prefLabel = LinkedData::Utils::Triples.last_iri_fragment c.id.to_s - end - - if @submission.ontology.viewOf.nil? - loomLabel = LinkedData::Models::OntologySubmission.loom_transform_literal(prefLabel.to_s) - - if loomLabel.length > 2 - artifacts[:mapping_triples] << LinkedData::Utils::Triples.loom_mapping_triple( - c.id, Goo.vocabulary(:metadata_def)[:mappingLoom], loomLabel) - end - artifacts[:mapping_triples] << LinkedData::Utils::Triples.uri_mapping_triple( - c.id, Goo.vocabulary(:metadata_def)[:mappingSameURI], c.id) - end - end - - def generate_missing_labels_post_page(artifacts = {}, logger, paging, page_classes, page) - rest_mappings = LinkedData::Mappings.migrate_rest_mappings(@submission.ontology.acronym) - artifacts[:mapping_triples].concat(rest_mappings) - - if artifacts[:label_triples].length > 0 - logger.info("Asserting #{artifacts[:label_triples].length} labels in " + - "#{@submission.id.to_ntriples}") - logger.flush - artifacts[:label_triples] = artifacts[:label_triples].join("\n") - artifacts[:fsave].write(artifacts[:label_triples]) - t0 = Time.now - Goo.sparql_data_client.append_triples(@submission.id, artifacts[:label_triples], mime_type = "application/x-turtle") - t1 = Time.now - logger.info("Labels asserted in #{t1 - t0} sec.") - logger.flush - else - logger.info("No labels generated in page #{page}.") - logger.flush - end - - if artifacts[:mapping_triples].length > 0 - logger.info("Asserting #{artifacts[:mapping_triples].length} mappings in " + - "#{@submission.id.to_ntriples}") - logger.flush - artifacts[:mapping_triples] = artifacts[:mapping_triples].join("\n") - artifacts[:fsave_mappings].write(artifacts[:mapping_triples]) - - t0 = Time.now - Goo.sparql_data_client.append_triples(@submission.id, artifacts[:mapping_triples], mime_type = "application/x-turtle") - t1 = Time.now - logger.info("Mapping labels asserted in #{t1 - t0} sec.") - logger.flush - end - end - - def generate_missing_labels_post(artifacts = {}, logger, pagging) - logger.info("end generate_missing_labels traversed #{artifacts[:count_classes]} classes") - logger.info("Saved generated labels in #{artifacts[:save_in_file]}") - artifacts[:fsave].close() - artifacts[:fsave_mappings].close() - logger.flush - end - - end - class SubmissionRDFGenerator < OntologySubmissionProcess def process(logger, options) @@ -319,7 +32,6 @@ def process_rdf(logger, reasoning) @submission.remove_submission_status(status) #remove RDF status before starting generate_rdf(logger, reasoning: reasoning) - @submission.extract_metadata @submission.add_submission_status(status) if @submission.valid? @@ -327,9 +39,8 @@ def process_rdf(logger, reasoning) @submission.previous_values.clear @submission.save else - logger.error("Submission is not valid after extracting metadata: #{@submission.errors.inspect}") + logger.error("Submission is not valid after processing RDF: #{@submission.errors.inspect}") end - rescue Exception => e logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") logger.flush @@ -346,22 +57,6 @@ def process_rdf(logger, reasoning) # If RDF generation fails, no point of continuing raise e end - - MissingLabelsHandler.new(@submission).process(logger, file_path: @submission.uploadFilePath.to_s) - - status = LinkedData::Models::SubmissionStatus.find('OBSOLETE').first - begin - generate_obsolete_classes(logger, @submission.uploadFilePath.to_s) - @submission.add_submission_status(status) - @submission.save - rescue Exception => e - logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") - logger.flush - @submission.add_submission_status(status.get_error_status) - @submission.save - # if obsolete fails the parsing fails - raise e - end end def generate_rdf(logger, reasoning: true) @@ -421,65 +116,6 @@ def delete_and_append(triples_file_path, logger, mime_type = nil) logger.flush end - def generate_obsolete_classes(logger, file_path) - @submission.bring(:obsoleteProperty) if @submission.bring?(:obsoleteProperty) - @submission.bring(:obsoleteParent) if @submission.bring?(:obsoleteParent) - classes_deprecated = [] - if @submission.obsoleteProperty && - @submission.obsoleteProperty.to_s != "http://www.w3.org/2002/07/owl#deprecated" - - predicate_obsolete = RDF::URI.new(@submission.obsoleteProperty.to_s) - query_obsolete_predicate = < 0 - classes_deprecated.uniq! - logger.info("Asserting owl:deprecated statement for #{classes_deprecated} classes") - save_in_file = File.join(File.dirname(file_path), "obsolete.ttl") - fsave = File.open(save_in_file, "w") - classes_deprecated.each do |class_id| - fsave.write(LinkedData::Utils::Triples.obselete_class_triple(class_id) + "\n") - end - fsave.close() - result = Goo.sparql_data_client.append_triples_from_file( - @submission.id, - save_in_file, - mime_type = "application/x-turtle") - end - end - end end end diff --git a/lib/ontologies_linked_data/services/submission_process/submission_processor.rb b/lib/ontologies_linked_data/services/submission_process/submission_processor.rb index 9e922494c..7fbf94cb6 100644 --- a/lib/ontologies_linked_data/services/submission_process/submission_processor.rb +++ b/lib/ontologies_linked_data/services/submission_process/submission_processor.rb @@ -23,9 +23,6 @@ def process(logger, options = nil) def process_submission(logger, options = {}) # Wrap the whole process so we can email results begin - archive, diff, index_commit, index_properties, - index_search, process_rdf, reasoning, run_metrics = get_options(options) - @submission.bring_remaining @submission.ontology.bring_remaining @@ -33,41 +30,35 @@ def process_submission(logger, options = {}) logger.flush LinkedData::Parser.logger = logger - if archive + if process_archive?(options) @submission.archive else - @submission.generate_rdf(logger, reasoning: reasoning) if process_rdf - parsed = @submission.ready?(status: [:rdf, :rdf_labels]) - if index_search - unless parsed - raise StandardError, "The submission #{@submission.ontology.acronym}/submissions/#{@submission.submissionId} + @submission.generate_rdf(logger, reasoning: process_reasoning?(options)) if process_rdf?(options) + + parsed = @submission.ready?(status: %i[rdf]) + + @submission = @submission.extract_metadata(logger, user_params: options[:params], heavy_extraction: extract_metadata?(options)) + + @submission.generate_missing_labels(logger) if generate_missing_labels?(options) + + @submission.generate_obsolete_classes(logger) if generate_obsolete_classes?(options) + + if !parsed && (index_search?(options) || index_properties?(options) || index_all_data?(options)) + raise StandardError, "The submission #{@submission.ontology.acronym}/submissions/#{@submission.submissionId} cannot be indexed because it has not been successfully parsed" - end - @submission.index(logger, commit: index_commit) end - if index_properties - unless parsed - raise Exception, "The properties for the submission #{@submission.ontology.acronym}/submissions/#{@submission.submissionId} - cannot be indexed because it has not been successfully parsed" + @submission.index_all(logger, commit: process_index_commit?(options)) if index_all_data?(options) - end - @submission.index_properties(logger, commit: index_commit) - end + @submission.index_terms(logger, commit: process_index_commit?(options)) if index_search?(options) - @submission.generate_diff(logger) if diff + @submission.index_properties(logger, commit: process_index_commit?(options)) if index_properties?(options) - if run_metrics - unless parsed - raise StandardError, "Metrics cannot be generated on the submission - #{@submission.ontology.acronym}/submissions/#{@submission.submissionId} - because it has not been successfully parsed" - end - @submission.generate_metrics(logger) - end - end + @submission.generate_metrics(logger) if process_metrics?(options) + @submission.generate_diff(logger) if process_diff?(options) + end @submission.save logger.info("Submission processing of #{@submission.id} completed successfully") logger.flush @@ -84,40 +75,54 @@ def notify_submission_processed(logger) logger.error("Email sending failed: #{e.message}\n#{e.backtrace.join("\n\t")}"); logger.flush end - def get_options(options) - - if options.empty? - process_rdf = true - index_search = true - index_properties = true - index_commit = true - run_metrics = true - reasoning = true - diff = true - archive = false - else - process_rdf = options[:process_rdf] == true - index_search = options[:index_search] == true - index_properties = options[:index_properties] == true - run_metrics = options[:run_metrics] == true - - reasoning = if !process_rdf || options[:reasoning] == false - false - else - true - end - - index_commit = if (!index_search && !index_properties) || options[:index_commit] == false - false - else - true - end - - diff = options[:diff] == true - archive = options[:archive] == true - end - [archive, diff, index_commit, index_properties, index_search, process_rdf, reasoning, run_metrics] + def process_archive?(options) + options[:archive].eql?(true) + end + + def process_rdf?(options) + options.empty? || options[:process_rdf].eql?(true) + end + + def generate_missing_labels?(options) + options[:generate_missing_labels].nil? && process_rdf?(options) || options[:generate_missing_labels].eql?(true) + end + + def generate_obsolete_classes?(options) + options[:generate_obsolete_classes].nil? && process_rdf?(options) || options[:generate_obsolete_classes].eql?(true) end + + def index_all_data?(options) + options.empty? || options[:index_all_data].eql?(true) + end + + def index_search?(options) + options.empty? || options[:index_search].eql?(true) + end + + def index_properties?(options) + options.empty? || options[:index_properties].eql?(true) + end + + def process_index_commit?(options) + index_search?(options) || index_properties?(options) || index_all_data?(options) + end + + def process_diff?(options) + options.empty? || options[:diff].eql?(true) + end + + def process_metrics?(options) + options.empty? || options[:run_metrics].eql?(true) + end + + def process_reasoning?(options) + options.empty? && options[:reasoning].eql?(true) + end + + def extract_metadata?(options) + options.empty? || options[:extract_metadata].eql?(true) + end + end end -end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/utils/file.rb b/lib/ontologies_linked_data/utils/file.rb index 5b6277f3d..956ca7b0a 100644 --- a/lib/ontologies_linked_data/utils/file.rb +++ b/lib/ontologies_linked_data/utils/file.rb @@ -152,7 +152,6 @@ def self.download_file(uri, limit = 10) file, filename = download_file_ftp(uri) else file = Tempfile.new('ont-rest-file') - file_size = 0 filename = nil http_session = Net::HTTP.new(uri.host, uri.port) http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE diff --git a/lib/ontologies_linked_data/utils/notifier.rb b/lib/ontologies_linked_data/utils/notifier.rb index 96d60da1d..aff6be7c0 100644 --- a/lib/ontologies_linked_data/utils/notifier.rb +++ b/lib/ontologies_linked_data/utils/notifier.rb @@ -73,7 +73,7 @@ def self.ontology_admin_emails(ontology) user.bring(:email) if user.bring?(:email) recipients << user.email end - recipients + recipients.sort end def self.ontoportal_admin_emails diff --git a/lib/ontologies_linked_data/utils/triples.rb b/lib/ontologies_linked_data/utils/triples.rb index 7a1385226..99ecbdbcc 100644 --- a/lib/ontologies_linked_data/utils/triples.rb +++ b/lib/ontologies_linked_data/utils/triples.rb @@ -1,3 +1,4 @@ + module LinkedData module Utils module Triples @@ -23,12 +24,12 @@ def self.rdf_for_custom_properties(ont_sub) # Add subPropertyOf triples for custom properties unless ont_sub.prefLabelProperty.nil? unless ont_sub.prefLabelProperty == Goo.vocabulary(:rdfs)[:label] || ont_sub.prefLabelProperty == Goo.vocabulary(:metadata_def)[:prefLabel] - triples << triple(ont_sub.prefLabelProperty, subPropertyOf, Goo.vocabulary(:metadata_def)[:prefLabel]) + triples << triple(ont_sub.prefLabelProperty, subPropertyOf, Goo.vocabulary(:metadata_def)[:prefLabel]) end end unless ont_sub.definitionProperty.nil? unless ont_sub.definitionProperty == Goo.vocabulary(:rdfs)[:label] || ont_sub.definitionProperty == Goo.vocabulary(:skos)[:definition] - triples << triple(ont_sub.definitionProperty, subPropertyOf, Goo.vocabulary(:skos)[:definition]) + triples << triple(ont_sub.definitionProperty, subPropertyOf, Goo.vocabulary(:skos)[:definition]) end end unless ont_sub.synonymProperty.nil? @@ -75,11 +76,11 @@ def self.label_for_class_triple(class_id, property, label, language=nil) params = { datatype: RDF::XSD.string } lang = language.to_s.downcase - if !lang.empty? && lang.to_sym != :none + if !lang.empty? && lang.to_sym != :none && !lang.to_s.eql?('@none') params[:datatype] = RDF.langString params[:language] = lang.to_sym end - return triple(class_id, property, RDF::Literal.new(label, params)) + triple(class_id, property, RDF::Literal.new(label, **params)) end def self.generated_label(class_id, existing_label) diff --git a/ontologies_linked_data.gemspec b/ontologies_linked_data.gemspec index bbc3dc99b..9498b4f2c 100644 --- a/ontologies_linked_data.gemspec +++ b/ontologies_linked_data.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |gem| gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } gem.require_paths = ["lib"] - gem.required_ruby_version = ">= 3.1" + gem.required_ruby_version = ">= 3.2" gem.add_dependency("activesupport") gem.add_dependency("bcrypt") @@ -32,6 +32,7 @@ Gem::Specification.new do |gem| gem.add_dependency("rack-test") gem.add_dependency("rsolr") gem.add_dependency("rubyzip", "~> 3.0") + gem.add_dependency("down", "~> 5.0") gem.add_development_dependency("email_spec") diff --git a/rakelib/ontoportal_testkit.rake b/rakelib/ontoportal_testkit.rake new file mode 100644 index 000000000..39a3ec73d --- /dev/null +++ b/rakelib/ontoportal_testkit.rake @@ -0,0 +1 @@ +require "ontoportal/testkit/tasks" diff --git a/test/data/ontology_files/BRO_v3.2.owl b/test/data/ontology_files/BRO_v3.2.owl index 247e35e63..4bd03f7a1 100644 --- a/test/data/ontology_files/BRO_v3.2.owl +++ b/test/data/ontology_files/BRO_v3.2.owl @@ -643,7 +643,7 @@ Activity related to the creation, use, or maintenance of a biorepository (http://en.wikipedia.org/wiki/Biorepository) Gestion des échantillons biologiques - Biospecimen Management + Gestion des Bioéchantillons @@ -719,6 +719,7 @@ As defined in http://en.wikipedia.org/wiki/Gene_therapy + Gene Therapy Thérapie génique diff --git a/test/data/ontology_files/apto.owl b/test/data/ontology_files/apto.owl new file mode 100644 index 000000000..ceb167c52 --- /dev/null +++ b/test/data/ontology_files/apto.owl @@ -0,0 +1,11147 @@ + + + + + + + + + + + + + + + + + + http://www.w3.org/TR/skos-reference/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + agrovoc:c_019ee1cc + Farelo de trigo + + + + agrovoc:c_019ee1cc + Wheat bran + + + + agrovoc:c_10097 + Coentro + + + + agrovoc:c_10097 + Coriander + + + + agrovoc:c_10097 + https://sistemas.sede.embrapa.br/agrotermos/resources/64eb2b6204d7163a62fd0692b221260e + + + + agrovoc:c_10195 + Cucumber + + + + agrovoc:c_10195 + Pepino + + + + agrovoc:c_10677 + Etanol + + + + agrovoc:c_10677 + Ethanol + + + + agrovoc:c_10677 + https://sistemas.sede.embrapa.br/agrotermos/resources/95eac3b4e952901f4b80aea54f6f9e16 + + + + agrovoc:c_1071 + Pão + + + + agrovoc:c_1071 + bread + + + + agrovoc:c_10722 + Broad beans + + + + agrovoc:c_10722 + Fava + + + + agrovoc:c_11091 + Alho + + + + agrovoc:c_11091 + Garlic + + + + agrovoc:c_11131 + Germe de trigo + + + + agrovoc:c_11131 + Wheat germ + + + + agrovoc:c_11392 + Goiaba + + + + agrovoc:c_11392 + Guava + + + + agrovoc:c_11392 + https://sistemas.sede.embrapa.br/agrotermos/resources/a0b91d8e06136094e4e9f2633d219324 + + + + agrovoc:c_1147 + Boi + + + + agrovoc:c_1147 + Bullock + + + + agrovoc:c_1147 + https://sistemas.sede.embrapa.br/agrotermos/resources/86a270a74dd36082d129dbc67da00c36 + + + + agrovoc:c_12109 + Galinha poedeira + + + + agrovoc:c_12109 + Layer chicken + + + + agrovoc:c_12109 + https://sistemas.sede.embrapa.br/agrotermos/resources/fd2c4059780b8a9637c4e67191980744 + + + + agrovoc:c_12151 + Alface + + + + agrovoc:c_12151 + Lettuce + + + + agrovoc:c_12151 + https://sistemas.sede.embrapa.br/agrotermos/resources/57a28dbfb65685d05aba506ff9728f85 + + + + agrovoc:c_1219 + Bezerro + + + + agrovoc:c_1219 + Calf + + + + agrovoc:c_1219 + https://sistemas.sede.embrapa.br/agrotermos/resources/dbe05e794688e1e0a690316f4e3cb155 + + + + agrovoc:c_12332 + Maize + + + + agrovoc:c_12332 + Milho + + + + agrovoc:c_12332 + https://sistemas.sede.embrapa.br/agrotermos/resources/48f22450ff89f732ae0cd7c0f2dfe1eb + + + + agrovoc:c_12367 + Manga + + + + agrovoc:c_12367 + Mango + + + + agrovoc:c_12367 + https://sistemas.sede.embrapa.br/agrotermos/resources/b16e21785bcdd2394b3e2a77de0f8eaa + + + + agrovoc:c_12487 + Melão + + + + agrovoc:c_12487 + Muskmelon + + + + agrovoc:c_12920 + Okra + + + + agrovoc:c_12920 + Quiabo + + + + agrovoc:c_12934 + Cebola + + + + agrovoc:c_12934 + Onion + + + + agrovoc:c_12934 + https://sistemas.sede.embrapa.br/agrotermos/resources/281d59c6d9c8fe4b4e627a3aa56a9670 + + + + agrovoc:c_13120 + Parsley + + + + agrovoc:c_13120 + Salsa + + + + agrovoc:c_13120 + Salsinha + + + + agrovoc:c_13120 + Salsa + + + + agrovoc:c_13132 + Leite pasteurizado + + + + agrovoc:c_13132 + Pasteurized milk + + + + agrovoc:c_13132 + https://sistemas.sede.embrapa.br/agrotermos/resources/6a3e0cb43f80ee57b0895dc0870d5f12 + + + + agrovoc:c_13394 + Abacaxi + + + + agrovoc:c_13394 + Pineapple + + + + agrovoc:c_13394 + https://sistemas.sede.embrapa.br/agrotermos/resources/4b96d5c1ff312eea069ddc760794963d + + + + agrovoc:c_13551 + Batata + + + + agrovoc:c_13551 + Potatoe + + + + agrovoc:c_14010 + Centeio + + + + agrovoc:c_14010 + Rye + + + + agrovoc:c_1423 + Cellulose + + + + agrovoc:c_1423 + Celulose + + + + agrovoc:c_1423 + https://sistemas.sede.embrapa.br/agrotermos/resources/6a324a803fcf2cb57faddfe4ea18326a + + + + agrovoc:c_14386 + Trigo mole + + + + agrovoc:c_14386 + soft wheat + + + + agrovoc:c_14386 + Trigo brando + + + + agrovoc:c_14386 + Trigo mole + + + + agrovoc:c_14477 + Soja + + + + agrovoc:c_14477 + Soybean + + + + agrovoc:c_14477 + Soja em Grão + + + + agrovoc:c_14477 + https://sistemas.sede.embrapa.br/agrotermos/resources/bdc7645e04618dd9f14f06d96ab6967d + + + + agrovoc:c_14727 + Pimentão + + + + agrovoc:c_14727 + Sweet pepper + + + + agrovoc:c_14727 + https://sistemas.sede.embrapa.br/agrotermos/resources/a5f74e850f12620e61ba8013e75b59b4 + + + + agrovoc:c_14729 + Batata-doce + + + + agrovoc:c_14729 + Sweet potatoe + + + + agrovoc:c_1473 + Cereal product + + + + agrovoc:c_1473 + Produto à base de cereal + + + + agrovoc:c_1473 + https://sistemas.sede.embrapa.br/agrotermos/resources/7f875377fcab131e0629c16fc503e31a + + + + agrovoc:c_1474 + Cereal + + + + agrovoc:c_1474 + Cereal + + + + agrovoc:c_1474 + https://sistemas.sede.embrapa.br/agrotermos/resources/42a1751fa85f0a5024720bd40f350ba3 + + + + agrovoc:c_14791 + Inhame + + + + agrovoc:c_14791 + Taro + + + + agrovoc:c_14791 + Cará + + + + agrovoc:c_14791 + Taro + + + + agrovoc:c_14791 + https://sistemas.sede.embrapa.br/agrotermos/resources/3886bc9c9655241c5737d45eec5447bb + + + + agrovoc:c_14791 + Inhame + + + + agrovoc:c_15068 + Leite UHT + + + + agrovoc:c_15068 + UHT milk + + + + agrovoc:c_15068 + https://sistemas.sede.embrapa.br/agrotermos/resources/2428d4ce74f58cd871700239f5136f3f + + + + agrovoc:c_1507 + Cheese + + + + agrovoc:c_1507 + Queijo + + + + agrovoc:c_1507 + https://sistemas.sede.embrapa.br/agrotermos/resources/c4d194cfc9f10a4885bd388564036bef + + + + agrovoc:c_15277 + Melancia + + + + agrovoc:c_15277 + Watermelon + + + + agrovoc:c_15277 + https://sistemas.sede.embrapa.br/agrotermos/resources/5e8f4e06a72887b71ce41638fca221cf + + + + agrovoc:c_1540 + Chicken + + + + agrovoc:c_1540 + Frango + + + + agrovoc:c_1540 + https://sistemas.sede.embrapa.br/agrotermos/resources/10d5a0555a83f8bb41290c1e14e603e3 + + + + agrovoc:c_15463 + Iogurte + + + + agrovoc:c_15463 + Yoghurt + + + + agrovoc:c_15463 + https://sistemas.sede.embrapa.br/agrotermos/resources/2a65973eca6cc9283ab62acf05a41ba2 + + + + agrovoc:c_15685 + Especiaria + + + + agrovoc:c_15685 + Spice + + + + agrovoc:c_15685 + https://sistemas.sede.embrapa.br/agrotermos/resources/8eb41f06f00ccb7e912de9c6782ccea4 + + + + agrovoc:c_15742 + Processed product + + + + agrovoc:c_15742 + Produto processado + + + + agrovoc:c_15742 + https://sistemas.sede.embrapa.br/agrotermos/resources/7d7cb76048f9f558389d3ef8c77f4e82 + + + + agrovoc:c_15978 + Carne caprina + + + + agrovoc:c_15978 + Goat meat + + + + agrovoc:c_15978 + Carne de caprino + + + + agrovoc:c_15978 + https://sistemas.sede.embrapa.br/agrotermos/resources/bf2ffd84f991885387456605b1decce0 + + + + agrovoc:c_15978 + Carne caprina + + + + agrovoc:c_16080 + Cow milk + + + + agrovoc:c_16080 + Leite de vaca + + + + agrovoc:c_16083 + Goat milk + + + + agrovoc:c_16083 + Leite de cabra + + + + agrovoc:c_16083 + https://sistemas.sede.embrapa.br/agrotermos/resources/6b1c99fc07dc0e133ab5b1282f5ed6ea + + + + agrovoc:c_1641 + Citrus fruit + + + + agrovoc:c_1641 + Fruta cítrica + + + + agrovoc:c_1641 + Citrus + + + + agrovoc:c_1641 + Fruta cítrica + + + + agrovoc:c_16477 + Açúcar de cana + + + + agrovoc:c_16477 + Cane sugar + + + + agrovoc:c_16477 + https://sistemas.sede.embrapa.br/agrotermos/resources/9852e4677bf923bfb2ac3b6136924a0b + + + + agrovoc:c_16508 + Soybean oil + + + + agrovoc:c_16508 + Óleo de soja + + + + agrovoc:c_1926 + Algodão + + + + agrovoc:c_1926 + Cotton + + + + agrovoc:c_1926 + https://sistemas.sede.embrapa.br/agrotermos/resources/d8f3d67c92743cb979008b6372af3176 + + + + agrovoc:c_1939 + Cow + + + + agrovoc:c_1939 + Vaca + + + + agrovoc:c_1939 + https://sistemas.sede.embrapa.br/agrotermos/resources/90f077d7759d0d4d21e6867727d4b2bd + + + + agrovoc:c_2384 + Dried milk + + + + agrovoc:c_2384 + Leite em pó + + + + agrovoc:c_2384 + https://sistemas.sede.embrapa.br/agrotermos/resources/7d3e6a0eddeeb429cd94658f5c6f2626 + + + + agrovoc:c_24000 + Carne de frango + + + + agrovoc:c_24000 + Chicken meat + + + + agrovoc:c_24000 + Carne de galinha + + + + agrovoc:c_24000 + https://sistemas.sede.embrapa.br/agrotermos/resources/5bdef85b388203f1f3aa24880672146c + + + + agrovoc:c_249 + Alcohol + + + + agrovoc:c_249 + Álcool + + + + agrovoc:c_249 + https://sistemas.sede.embrapa.br/agrotermos/resources/79b89ad67167fad67c264a54ef962b44 + + + + agrovoc:c_2502 + Egg + + + + agrovoc:c_2502 + Ovo + + + + agrovoc:c_2502 + https://sistemas.sede.embrapa.br/agrotermos/resources/8472071ced24fe2226f7ca33cff91272 + + + + agrovoc:c_25470 + Coco-da-baía + + + + agrovoc:c_25470 + Coconut + + + + agrovoc:c_25470 + Coco + + + + agrovoc:c_25473 + Cottonseed + + + + agrovoc:c_25473 + Semente de algodão + + + + agrovoc:c_25473 + Caroço de algodão + + + + agrovoc:c_25473 + https://sistemas.sede.embrapa.br/agrotermos/resources/a0eb8f32cd10ba23677e89f9e871ef04 + + + + agrovoc:c_25473 + Semente de algodão + + + + agrovoc:c_25490 + Oil seed + + + + agrovoc:c_25490 + Semente oleaginosa + + + + agrovoc:c_25490 + Oleaginosas + + + + agrovoc:c_25490 + https://sistemas.sede.embrapa.br/agrotermos/resources/11d1922eb5e932db908581eb83e3f457 + + + + agrovoc:c_25499 + Rapeseed + + + + agrovoc:c_25499 + Semente de canola + + + + agrovoc:c_25499 + Canola + + + + agrovoc:c_25499 + Semente de colza + + + + agrovoc:c_25499 + https://sistemas.sede.embrapa.br/agrotermos/resources/695b2bf6e87d16f2981ae004561f7ea8 + + + + agrovoc:c_25499 + Semente de canola + + + + agrovoc:c_25507 + Semente de girassol + + + + agrovoc:c_25507 + Sunflower seed + + + + agrovoc:c_25507 + Girassol + + + + agrovoc:c_25507 + https://sistemas.sede.embrapa.br/agrotermos/resources/f7ce8e4b85f4e1bd909d8ca66e203db0 + + + + agrovoc:c_25507 + Semente de girassol + + + + agrovoc:c_25511 + Farinha de trigo + + + + agrovoc:c_25511 + Wheat flour + + + + agrovoc:c_26767 + Dairy cow + + + + agrovoc:c_26767 + Vaca leiteira + + + + agrovoc:c_26767 + https://sistemas.sede.embrapa.br/agrotermos/resources/99c85c3f0a8cf76f365de195ab35f8a3 + + + + agrovoc:c_28379 + Café em grãos + + + + agrovoc:c_28379 + Coffee beans + + + + agrovoc:c_28379 + Café + + + + agrovoc:c_28379 + Café em grãos + + + + agrovoc:c_29108 + Animal útil + + + + agrovoc:c_29108 + Useful animal + + + + agrovoc:c_29108 + https://sistemas.sede.embrapa.br/agrotermos/resources/ba94e65756b89654616f98cce03aee8a + + + + agrovoc:c_2943 + Fish + + + + agrovoc:c_2943 + Peixe + + + + agrovoc:c_2943 + https://sistemas.sede.embrapa.br/agrotermos/resources/cb4bb0dafca8dc9e0452723bc627435d + + + + agrovoc:c_2988 + Farinha + + + + agrovoc:c_2988 + Flour + + + + agrovoc:c_2988 + https://sistemas.sede.embrapa.br/agrotermos/resources/9df38bda0adee40b17d54026429d0fdb + + + + agrovoc:c_29989 + Processed plant product + + + + agrovoc:c_29989 + Produto de origem vegetal processado + + + + agrovoc:c_3049 + Forest product + + + + agrovoc:c_3049 + Produto florestal + + + + agrovoc:c_3105 + Freshwater fishe + + + + agrovoc:c_3105 + Peixe de água doce + + + + agrovoc:c_3131 + Fruit + + + + agrovoc:c_3131 + Fruta + + + + agrovoc:c_3131 + https://sistemas.sede.embrapa.br/agrotermos/resources/233926cc509b0d57b7188653097d1eec + + + + agrovoc:c_32720 + Tilapia + + + + agrovoc:c_32720 + Tilápia + + + + agrovoc:c_32720 + https://sistemas.sede.embrapa.br/agrotermos/resources/a30d5e4e0ec8526c396994bd8c4e637b + + + + agrovoc:c_331566 + Beans + + + + agrovoc:c_331566 + Feijão + + + + agrovoc:c_331566 + https://sistemas.sede.embrapa.br/agrotermos/resources/62b8288c2867583bbbfc172290439710 + + + + agrovoc:c_3324 + Caprino + + + + agrovoc:c_3324 + Goat + + + + agrovoc:c_3324 + https://sistemas.sede.embrapa.br/agrotermos/resources/fb00d24ff5fe1e47871b7876841f5024 + + + + agrovoc:c_3346 + Grain + + + + agrovoc:c_3346 + Grão + + + + agrovoc:c_3346 + https://sistemas.sede.embrapa.br/agrotermos/resources/b868e45362655d1998f3bee01f991d94 + + + + agrovoc:c_3359 + Uva + + + + agrovoc:c_3359 + grape + + + + agrovoc:c_3359 + https://sistemas.sede.embrapa.br/agrotermos/resources/d3da5eb0da6870aec2297a935d7159a8 + + + + agrovoc:c_3376 + Hortaliça de folha + + + + agrovoc:c_3376 + Leaf vegetable + + + + agrovoc:c_3376 + https://sistemas.sede.embrapa.br/agrotermos/resources/33ad8f317947f7c5a1b849e49d5483e2 + + + + agrovoc:c_3376 + Folhosas + + + + agrovoc:c_3376 + Hortaliça de folha + + + + agrovoc:c_3609 + Couros e peles + + + + agrovoc:c_3609 + Hides and skins + + + + agrovoc:c_3609 + https://sistemas.sede.embrapa.br/agrotermos/resources/378cb44a9af7ef24dff9873abb5fb8b2 + + + + agrovoc:c_36410 + Açúcar refinado + + + + agrovoc:c_36410 + Refined sugar + + + + agrovoc:c_36410 + https://sistemas.sede.embrapa.br/agrotermos/resources/891c923495e2fae8b1a751afb2956895 + + + + agrovoc:c_3652 + Honey + + + + agrovoc:c_3652 + Mel de abelha + + + + agrovoc:c_3652 + https://sistemas.sede.embrapa.br/agrotermos/resources/8f50ba9e7d863515cdc192cfdce1eb41 + + + + agrovoc:c_36538 + Casulo de seda + + + + agrovoc:c_36538 + Cocoon + + + + agrovoc:c_36538 + Casulo + + + + agrovoc:c_36538 + https://sistemas.sede.embrapa.br/agrotermos/resources/3fb7f54dc80462112da0aaf9e280a5d3 + + + + agrovoc:c_36538 + Casulo de seda + + + + agrovoc:c_3654 + Abelha melífera + + + + agrovoc:c_3654 + Honey bee + + + + agrovoc:c_3654 + Abelha produtora de mel + + + + agrovoc:c_3654 + Abelha melífera + + + + agrovoc:c_37735 + Abóbora + + + + agrovoc:c_37735 + Squashe + + + + agrovoc:c_37735 + https://sistemas.sede.embrapa.br/agrotermos/resources/0641ab8e01b7f3b5fe5c340082784021 + + + + agrovoc:c_3881 + Composto inorgânico + + + + agrovoc:c_3881 + Inorganic compound + + + + agrovoc:c_3881 + https://sistemas.sede.embrapa.br/agrotermos/resources/9bc1d45dc6914580199935a8c1e26709 + + + + agrovoc:c_4065 + Juta + + + + agrovoc:c_4065 + Jute + + + + agrovoc:c_4070 + Couve + + + + agrovoc:c_4070 + Kale + + + + agrovoc:c_4070 + https://sistemas.sede.embrapa.br/agrotermos/resources/5779a2c5efecaa6c45833f93794286e1 + + + + agrovoc:c_4163 + Cordeiro + + + + agrovoc:c_4163 + Lamb + + + + agrovoc:c_4163 + https://sistemas.sede.embrapa.br/agrotermos/resources/8731dc7e812594c5353915ed54fbb5ba + + + + agrovoc:c_4241 + Couro + + + + agrovoc:c_4241 + Leather + + + + agrovoc:c_4259 + Lemon + + + + agrovoc:c_4259 + Limão + + + + agrovoc:c_4259 + https://sistemas.sede.embrapa.br/agrotermos/resources/23a6a6f019e446235d9251e7bf1939c4 + + + + agrovoc:c_438 + Animal product + + + + agrovoc:c_438 + Produto de origem animal + + + + agrovoc:c_438 + https://sistemas.sede.embrapa.br/agrotermos/resources/ec624cf8e3122d30f5c35286cfb3efc7 + + + + agrovoc:c_4397 + Gado + + + + agrovoc:c_4397 + Livestock + + + + agrovoc:c_4647 + Mate + + + + agrovoc:c_4647 + Mate + + + + agrovoc:c_4647 + Erva mate + + + + agrovoc:c_4647 + https://sistemas.sede.embrapa.br/agrotermos/resources/7c13aea47d6e6ddefd62d2c00653b2a4 + + + + agrovoc:c_4647 + Mate + + + + agrovoc:c_4669 + Carne + + + + agrovoc:c_4669 + Meat + + + + agrovoc:c_4669 + https://sistemas.sede.embrapa.br/agrotermos/resources/409f4cd5e6a103f5a0d4bfcc18215b0e + + + + agrovoc:c_4826 + Leite + + + + agrovoc:c_4826 + Milk + + + + agrovoc:c_4827 + Milk by-product + + + + agrovoc:c_4827 + Subproduto do leite + + + + agrovoc:c_4827 + https://sistemas.sede.embrapa.br/agrotermos/resources/bea2cf5e51ed952a0f398806ac737878 + + + + agrovoc:c_4830 + Milk product + + + + agrovoc:c_4830 + Produto derivado do leite + + + + agrovoc:c_4830 + Dairy product + + + + agrovoc:c_4830 + https://sistemas.sede.embrapa.br/agrotermos/resources/124628c533f0720ccf575553b4feff1a + + + + agrovoc:c_4830 + Milk product + + + + agrovoc:c_4838 + Milheto + + + + agrovoc:c_4838 + millet + + + + agrovoc:c_4838 + mileto + + + + agrovoc:c_4838 + milhete + + + + agrovoc:c_5015 + Carne de carneiro + + + + agrovoc:c_5015 + Mutton + + + + agrovoc:c_5015 + https://sistemas.sede.embrapa.br/agrotermos/resources/2ed585b38f8e1a8e5c7278354ea40e11 + + + + agrovoc:c_5066 + Cabra + + + + agrovoc:c_5066 + Nannygoat + + + + agrovoc:c_5066 + https://sistemas.sede.embrapa.br/agrotermos/resources/7c21a0ecb0e51b244a6ba16f35a1047f + + + + agrovoc:c_5287 + Aveia + + + + agrovoc:c_5287 + Oat + + + + agrovoc:c_5287 + https://sistemas.sede.embrapa.br/agrotermos/resources/b54fb2fcde17a9c2da33470ae95fe39f + + + + agrovoc:c_5330 + Oil palm + + + + agrovoc:c_5330 + Palmeira Oleaginosa + + + + agrovoc:c_5330 + https://sistemas.sede.embrapa.br/agrotermos/resources/821d61281118d169f36bb25a77c5fe48 + + + + agrovoc:c_541 + Maçã + + + + agrovoc:c_541 + apple + + + + agrovoc:c_5548 + Papel + + + + agrovoc:c_5548 + Paper + + + + agrovoc:c_5548 + https://sistemas.sede.embrapa.br/agrotermos/resources/acbf61dc603a375f7841d5bfd5ca928f + + + + agrovoc:c_5638 + Pêssego + + + + agrovoc:c_5638 + peache + + + + agrovoc:c_566 + Arabica coffee + + + + agrovoc:c_566 + Café arábica + + + + agrovoc:c_5c9e76bc + Rúcula + + + + agrovoc:c_5c9e76bc + rocket + + + + agrovoc:c_5c9e76bc + https://sistemas.sede.embrapa.br/agrotermos/resources/8c6d70bd7e4b4f0fa5cca06e224ac3ba + + + + agrovoc:c_6120 + Carne suína + + + + agrovoc:c_6120 + Pork + + + + agrovoc:c_6120 + Carne de porco + + + + agrovoc:c_6120 + https://sistemas.sede.embrapa.br/agrotermos/resources/03ff0226008ff0fe87cf33873a786fb0 + + + + agrovoc:c_6120 + Carne suína + + + + agrovoc:c_6145 + Ave de capoeira + + + + agrovoc:c_6145 + Poultry + + + + agrovoc:c_6599 + Arroz + + + + agrovoc:c_6599 + Rice + + + + agrovoc:c_6599 + https://sistemas.sede.embrapa.br/agrotermos/resources/cd762ef822be9c0c8b1d8b2c3633eddf + + + + agrovoc:c_6626 + Café robusta + + + + agrovoc:c_6626 + Robusta coffee + + + + agrovoc:c_6678 + Borracha + + + + agrovoc:c_6678 + Rubber + + + + agrovoc:c_6678 + Borracha natural + + + + agrovoc:c_6678 + https://sistemas.sede.embrapa.br/agrotermos/resources/2b5b436e38e557be6333ee663cc234f9 + + + + agrovoc:c_6678 + Borracha + + + + agrovoc:c_6679 + Planta produtora de borracha + + + + agrovoc:c_6679 + Rubber crops + + + + agrovoc:c_6679 + https://sistemas.sede.embrapa.br/agrotermos/resources/028e219a04f2febda150fadb1d886712 + + + + agrovoc:c_6ffce1ab + Fubá + + + + agrovoc:c_6ffce1ab + Maize meal + + + + agrovoc:c_6ffce1ab + Fubá de milho + + + + agrovoc:c_6ffce1ab + https://sistemas.sede.embrapa.br/agrotermos/resources/f9c9caaa48da729d2a8c0328388e9565 + + + + agrovoc:c_6ffce1ab + Fubá + + + + agrovoc:c_7030 + Ovino + + + + agrovoc:c_7030 + sheep + + + + agrovoc:c_7030 + https://sistemas.sede.embrapa.br/agrotermos/resources/d7c7bd6f1ed5b065106e6cb98cbce396 + + + + agrovoc:c_7064 + Bicho-da-seda + + + + agrovoc:c_7064 + Silkworm + + + + agrovoc:c_7086 + Sisal + + + + agrovoc:c_7086 + Sisal + + + + agrovoc:c_7086 + https://sistemas.sede.embrapa.br/agrotermos/resources/941490e4a8761000a77ebdd4b7f3a54d + + + + agrovoc:c_7249 + Sorghum grain + + + + agrovoc:c_7249 + Sorgo + + + + agrovoc:c_7249 + Sorgo granífero + + + + agrovoc:c_7249 + https://sistemas.sede.embrapa.br/agrotermos/resources/70e2bc0a07054af305aa5c113797a240 + + + + agrovoc:c_7249 + Sorgo + + + + agrovoc:c_7413 + Planta estimulante + + + + agrovoc:c_7413 + Stimulant crop + + + + agrovoc:c_7413 + https://sistemas.sede.embrapa.br/agrotermos/resources/8cfa9f207f98fd89447017bc69313888 + + + + agrovoc:c_7498 + Açúcar + + + + agrovoc:c_7498 + Sugar + + + + agrovoc:c_7498 + https://sistemas.sede.embrapa.br/agrotermos/resources/f34ae1525ab8cba9dfe78de135ac81f0 + + + + agrovoc:c_7550 + Laranja + + + + agrovoc:c_7550 + Sweet orange + + + + agrovoc:c_7550 + https://sistemas.sede.embrapa.br/agrotermos/resources/cf75ceb29197f57b19dcb8b4757368e8 + + + + agrovoc:c_7555 + Suíno + + + + agrovoc:c_7555 + Swine + + + + agrovoc:c_7555 + https://sistemas.sede.embrapa.br/agrotermos/resources/91125fa713391ed5b8fcdf4ca818d011 + + + + agrovoc:c_7601 + Tangerina + + + + agrovoc:c_7601 + Tangerine + + + + agrovoc:c_7601 + Bergamota + + + + agrovoc:c_7601 + Mandarina + + + + agrovoc:c_7601 + Mexerica + + + + agrovoc:c_7601 + https://sistemas.sede.embrapa.br/agrotermos/resources/d7b920215594108bd28b35702cb27236 + + + + agrovoc:c_7655 + Fruta de clima temperado + + + + agrovoc:c_7655 + Temperate fruit + + + + agrovoc:c_7805 + Tomate + + + + agrovoc:c_7805 + Tomatoe + + + + agrovoc:c_7805 + https://sistemas.sede.embrapa.br/agrotermos/resources/02e409c8bf00d35d7a7d61c56829da7f + + + + agrovoc:c_785 + Baked good + + + + agrovoc:c_785 + Produto de panificação + + + + agrovoc:c_7974 + Fruta tropical + + + + agrovoc:c_7974 + Tropical fruit + + + + agrovoc:c_7974 + https://sistemas.sede.embrapa.br/agrotermos/resources/bfb9ab9aadfc6cb021ca3d04221efc65 + + + + agrovoc:c_806 + Banana + + + + agrovoc:c_806 + Banana + + + + agrovoc:c_806 + https://sistemas.sede.embrapa.br/agrotermos/resources/72b302bf297a228a75730123efef7c41 + + + + agrovoc:c_8115 + Artrópode útil + + + + agrovoc:c_8115 + Useful arthropod + + + + agrovoc:c_8170 + Plant oil + + + + agrovoc:c_8170 + Óleo vegetal + + + + agrovoc:c_8170 + https://sistemas.sede.embrapa.br/agrotermos/resources/f3577d6a7af8c40945365082dd588456 + + + + agrovoc:c_8171 + Plant product + + + + agrovoc:c_8171 + Produto de origem vegetal + + + + agrovoc:c_8171 + https://sistemas.sede.embrapa.br/agrotermos/resources/67abbd6fd76a07e292828e3d15b34100 + + + + agrovoc:c_8174 + Produto hortícola + + + + agrovoc:c_8174 + Vegetable + + + + agrovoc:c_823 + Barley + + + + agrovoc:c_823 + Cevada + + + + agrovoc:c_8335 + Planta produtora de cera + + + + agrovoc:c_8335 + Wax plant + + + + agrovoc:c_8335 + https://sistemas.sede.embrapa.br/agrotermos/resources/a5528d684a04bc1651f0879ed04817d4 + + + + agrovoc:c_8373 + Trigo + + + + agrovoc:c_8373 + Wheat + + + + agrovoc:c_8373 + https://sistemas.sede.embrapa.br/agrotermos/resources/e2369eaf443fc28d5b546a7e23c7ea6c + + + + agrovoc:c_8376 + Soro do leite + + + + agrovoc:c_8376 + Whey + + + + agrovoc:c_8376 + https://sistemas.sede.embrapa.br/agrotermos/resources/9cd7fc78880bea64f045a006720f4c6d + + + + agrovoc:c_8421 + Madeira + + + + agrovoc:c_8421 + wood + + + + agrovoc:c_8421 + https://sistemas.sede.embrapa.br/agrotermos/resources/d471f65a3e7768cb32d51debc22d9f60 + + + + agrovoc:c_861 + Beef + + + + agrovoc:c_861 + Carne bovina + + + + agrovoc:c_861 + Carne de bovino + + + + agrovoc:c_861 + https://sistemas.sede.embrapa.br/agrotermos/resources/69c63ae967a1055e2b279791e2feedc0 + + + + agrovoc:c_861 + Carne bovina + + + + agrovoc:c_8998 + Berinjela + + + + agrovoc:c_8998 + Eggplant + + + + agrovoc:c_8998 + https://sistemas.sede.embrapa.br/agrotermos/resources/561a32dd09056431c368b5ad71e64432 + + + + agrovoc:c_9410 + Brazil nut + + + + agrovoc:c_9410 + Castanha do Brasil + + + + agrovoc:c_9410 + Castanha do Pará + + + + agrovoc:c_9410 + https://sistemas.sede.embrapa.br/agrotermos/resources/d8d964d1d7dbc6aab32e0ea56331e525 + + + + agrovoc:c_9410 + Castanha do Brasil + + + + agrovoc:c_9640 + Carrot + + + + agrovoc:c_9640 + Cenoura + + + + agrovoc:c_9640 + https://sistemas.sede.embrapa.br/agrotermos/resources/1a868182fce986fe0701a2d5c593acbf + + + + agrovoc:c_9649 + Cassava + + + + agrovoc:c_9649 + Mandioca + + + + agrovoc:c_9649 + Aipim + + + + agrovoc:c_9649 + Macaxeira + + + + agrovoc:c_9649 + raiz de mandioca + + + + agrovoc:c_9649 + https://sistemas.sede.embrapa.br/agrotermos/resources/373b74610542263335c865a6e39ee9d5 + + + + agrovoc:c_9649 + Mandioca + + + + agrovoc:c_9813 + Cebolinha + + + + agrovoc:c_9813 + Chives + + + + agrovoc:c_9813 + Green onion + + + + agrovoc:c_9813 + https://sistemas.sede.embrapa.br/agrotermos/resources/c0304fb7306cc510b82e5a98d7db7618 + + + + agrovoc:c_995ecefb + Planta útil + + + + agrovoc:c_995ecefb + Useful plant + + + + http://purl.obolibrary.org/obo/BCO_0000087 + A shortcut relation from an organsimal entity to a taxon. + + + + http://purl.obolibrary.org/obo/BCO_0000087 + Uma relação abreviada de uma entidade organismal para um táxon. + + + + http://purl.obolibrary.org/obo/BCO_0000087 + member of taxon + + + + http://purl.obolibrary.org/obo/BCO_0000087 + membro do táxon + + + + http://purl.obolibrary.org/obo/RO_0001000 + A relation between two distinct material entities, the new entity and the old entity, in which the new entity begins to exist when the old entity ceases to exist, and the new entity inherits the significant portion of the matter of the old entity. + + + + http://purl.obolibrary.org/obo/RO_0001000 + Uma relação entre duas entidades materiais distintas, a nova entidade e a entidade antiga, na qual a nova entidade começa a existir quando a entidade antiga deixa de existir, e a nova entidade herda a porção significativa da matéria da entidade antiga. + + + + http://purl.obolibrary.org/obo/RO_0001000 + derivado de + + + + http://purl.obolibrary.org/obo/RO_0001000 + derives from + + + + http://purl.obolibrary.org/obo/RO_0001001 + A relation between two distinct material entities, the old entity and the new entity, in which the new entity begins to exist when the old entity ceases to exist, and the new entity inherits the significant portion of the matter of the old entity. + + + + http://purl.obolibrary.org/obo/RO_0001001 + Uma relação entre duas entidades materiais distintas, a entidade antiga e a nova entidade, na qual a nova entidade começa a existir quando a entidade antiga deixa de existir, e a nova entidade herda a porção significativa da matéria da entidade antiga. + + + + http://purl.obolibrary.org/obo/RO_0001001 + deriva em + + + + http://purl.obolibrary.org/obo/RO_0001001 + derives into + + + + http://purl.obolibrary.org/obo/RO_0003000 + A produces B if some process that occurs_in A has_output B, where A and B are material entities. + + + + http://purl.obolibrary.org/obo/RO_0003000 + A produz B se algum processo que ocorre em A tem B como resultado, onde A e B são entidades materiais. + + + + http://purl.obolibrary.org/obo/RO_0003000 + produces + + + + http://purl.obolibrary.org/obo/RO_0003000 + produz + + + + http://purl.obolibrary.org/obo/RO_0003001 + A produced_by B iff some process that occurs_in B has_output A. + + + + http://purl.obolibrary.org/obo/RO_0003001 + A é produzido por B se, e somente se, algum processo que ocorre em B tem A como resultado. + + + + http://purl.obolibrary.org/obo/RO_0003001 + produced by + + + + http://purl.obolibrary.org/obo/RO_0003001 + produzido por + + + + http://www.w3.org/2004/02/skos/mapping + This vocabulary allows you to express information about how statements made using concepts from some conceptual scheme may be transformed into statements with concepts from a different scheme. So for example it can be used for automated query transformation. Or It could be used to create a virtual subject index for a collection in terms of an alternative classification system. + + + + http://www.w3.org/2004/02/skos/mapping + http://www.w3.org/2001/sw/Europe/reports/thes/8.4/ + + + + skos:altLabel + The range of skos:altLabel is the class of RDF plain literals. + + + + skos:altLabel + skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties. + + + + skos:altLabel + http://www.w3.org/2004/02/skos/core + + + + skos:altLabel + alternative label + + + + skos:broadMatch + If 'concept A has-broad-match concept B' then the set of resources properly indexed against concept A is a subset of the set of resources properly indexed against concept B. + + + + skos:broadMatch + skos:broadMatch + + + + skos:exactMatch + If two concepts are an 'exact-match' then the set of resources properly indexed against the first concept is identical to the set of resources properly indexed against the second. Therefore the two concepts may be interchanged in queries and subject-based indexes. + + + + skos:exactMatch + skos:exactMatch + + + + skos:hiddenLabel + The range of skos:hiddenLabel is the class of RDF plain literals. + + + + skos:hiddenLabel + skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties. + + + + skos:hiddenLabel + http://www.w3.org/2004/02/skos/core + + + + skos:hiddenLabel + hidden label + + + + skos:narrowMatch + If 'concept A has-narrow-match concept B' then the set of resources properly indexed against concept A is a superset of the set of resources properly indexed against concept B. + + + + skos:narrowMatch + skos:narrowMatch + + + + skos:prefLabel + A resource has no more than one value of skos:prefLabel per language tag, and no more than one value of skos:prefLabel without language tag. + + + + skos:prefLabel + The range of skos:prefLabel is the class of RDF plain literals. + + + + skos:prefLabel + skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise + disjoint properties. + + + + skos:prefLabel + http://www.w3.org/2004/02/skos/core + + + + skos:prefLabel + preferred label + + + + https://agrovoc.fao.org/browse/agrovoc/en/page/c_3568 + Culinary herbs + + + + https://agrovoc.fao.org/browse/agrovoc/en/page/c_3568 + Planta aromática para culinária + + + + https://agrovoc.fao.org/browse/agrovoc/en/page/c_7501 + Cana-de-açúcar + + + + https://agrovoc.fao.org/browse/agrovoc/en/page/c_7501 + Sugar cane + + + + https://agrovoc.fao.org/browse/agrovoc/en/page/c_7501 + https://sistemas.sede.embrapa.br/agrotermos/resources/fe91bcfb4bc023da0ecfabf5030f92db + + + + sdo:Açucar_Refinado_Amorfo + Açúcar refinado amorfo + + + + agrotermos:a110c0bb4ca441a61349d22192c6c583 + Piaçava + + + + agrotermos:aaff761df6b25c60c9454df207578876 + Amêndoa + + + + agrotermos:b675818cfcefea83a15d92ef70a73177 + Noz + + + + agrotermos:b675818cfcefea83a15d92ef70a73177 + Walnuts + + + + agrotermos:b675818cfcefea83a15d92ef70a73177 + noz inglesa + + + + agrotermos:b675818cfcefea83a15d92ef70a73177 + noz persa + + + + agrotermos:b675818cfcefea83a15d92ef70a73177 + http://aims.fao.org/aos/agrovoc/c_15254 + + + + agrotermos:b8019c5cd743abe5e568bca34e1035a2 + Amendoim + + + + agrotermos:c149abe6c0478ea98091733724878114 + Açúcar cristal + + + + agrotermos:c149abe6c0478ea98091733724878114 + Crystal sugar + + + + agrotermos:c7bd43872382af7e405eb31fbe0d62b3 + Pacu + + + + agrotermos:d624e0944dcd81829d9fcc3cc6611929 + Murici + + + + agrotermos:d624e0944dcd81829d9fcc3cc6611929 + murici-da-praia + + + + agrotermos:d624e0944dcd81829d9fcc3cc6611929 + murici-do-brejo + + + + agrotermos:d624e0944dcd81829d9fcc3cc6611929 + muruci + + + + agrotermos:d673d8cbfcb7c2fd26ef7fbd8b69d709 + Jenipapo + + + + agrotermos:d7f09ae1740a8f52cefd25b0614965a1 + Fécula + + + + agrotermos:d942eb37336785079bc6623ee5ff81a9 + Abóbora-menina + + + + agrotermos:d942eb37336785079bc6623ee5ff81a9 + Abobrinha + + + + agrotermos:d942eb37336785079bc6623ee5ff81a9 + abobrinha-italiana + + + + agrotermos:e4a3d44a48cc1650a37dc45948407a83 + Babaçu + + + + agrotermos:e59b534a1f0b7dc770e1b7578597c131 + Leguminosa de grão + + + + agrotermos:e59b534a1f0b7dc770e1b7578597c131 + Leguminosa de grão + + + + agrotermos:f52a78f322bd8fcd941bf77dee403436 + Mangaba + + + + agrotermos:fe46183f893373aeed0a008b54d71148 + Pinhão + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/0b251a6dcde4c65670b467527c2e12f3 + Fruta-do-conde + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/0b251a6dcde4c65670b467527c2e12f3 + Ata + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/0b251a6dcde4c65670b467527c2e12f3 + Pinha + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/05c55a77a73e2b5b498d15bea6fe6e35 + Novilho + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/1fabaa992654f45f4602e2f843463d50 + Ceriguela + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/1fabaa992654f45f4602e2f843463d50 + seriguela + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/1fabaa992654f45f4602e2f843463d50 + Ceriguela + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/2dc82a09c0941cb977352c10fba56d86 + Curimatã + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/28156081ffd59c01e5ad8c39d92d517b + Edible nut + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/28156081ffd59c01e5ad8c39d92d517b + Noz comestível + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/411edbea36c30f435e4d70751a9be9b2 + Planta produtora de madeira + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/411edbea36c30f435e4d70751a9be9b2 + Wood-producing plant + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/473830ed2508bec0622e627d7f8f9604 + Caju + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/473830ed2508bec0622e627d7f8f9604 + Cashew + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/473830ed2508bec0622e627d7f8f9604 + http://aims.fao.org/aos/agrovoc/c_9647 + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/5ca3ba89b5052310f056bffc84ad2359 + Pequi + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/5fe1bb8cb4a85f7794a509eebfc271a7 + Planta produtora de droga + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/51c17a606edc3755c249bd9f44a88a29 + Pirarucu + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/513d52ec57a549910bf9ced35be817a8 + Macauba Palm + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/513d52ec57a549910bf9ced35be817a8 + Macaúba + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/57285571deec8d096491da6f2bf7f2a6 + Castanha + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/57285571deec8d096491da6f2bf7f2a6 + http://aims.fao.org/aos/agrovoc/c_12873 + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/57285571deec8d096491da6f2bf7f2a6 + Nut + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/5949bdd80605dd4136a6b5a2cbe5b8f7 + Arroz em casca + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/5973875894f680f6171765597c0b302c + Vaca de corte + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6c577516edc7117699a72d620847be73 + Tabaco + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6c577516edc7117699a72d620847be73 + Tobacco + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6c577516edc7117699a72d620847be73 + fumo + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6c577516edc7117699a72d620847be73 + http://aims.fao.org/aos/agrovoc/c_7117 + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6ccfec83258bc84dd914b82963ff15a0 + Triticale + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6ccfec83258bc84dd914b82963ff15a0 + Triticale + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/6e5ec1f7742bbbfaccd577c26aa30bc5 + Macadâmia + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/61ff626b059dc368d047e9c27ede8eae + Cashew nut + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/61ff626b059dc368d047e9c27ede8eae + Castanha de caju + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/61ff626b059dc368d047e9c27ede8eae + https://agrovoc.fao.org/browse/agrovoc/en/page/c_9647 + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62ebfd337d11d466ca77260d335c62e6 + Mamão + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62ebfd337d11d466ca77260d335c62e6 + Papaia + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62646d1e38e30e9a1901a3cc69d75178 + Assai + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62646d1e38e30e9a1901a3cc69d75178 + Açaí + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62646d1e38e30e9a1901a3cc69d75178 + Juçara + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/62646d1e38e30e9a1901a3cc69d75178 + Açaí + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/638c80419412ec2aa5a3ab46897fcb04 + Bovine + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/638c80419412ec2aa5a3ab46897fcb04 + Bovino + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7a0149773ebabe1d82eb9830a577d16a + Acerola + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7a0149773ebabe1d82eb9830a577d16a + Barbados cherry + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7e8c918aba66629d9ac84d99fd56b745 + Umbu + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7794fc517590053809f758b7e16d87ed + Sal + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7794fc517590053809f758b7e16d87ed + Salt + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/7794fc517590053809f758b7e16d87ed + http://aims.fao.org/aos/agrovoc/c_33129 + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/8a1700ad77be316e1185a58a9df4663a + Suco + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/845caef614387a897ba1f99e3a3558cd + Fruit pulp + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/845caef614387a897ba1f99e3a3558cd + Polpa de fruta + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/9ddefc424391e81b5e0cea3f94bd04bd + Carnaúba + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/91e839283cf386499582d297bf181227 + Cumbaru + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/91e839283cf386499582d297bf181227 + Baru + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/91e839283cf386499582d297bf181227 + Cumaru + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/93091ce029083f6c99e904493ae41517 + Cupuaçu + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/936400f151ba2146a86cfcc342279f57 + Cajá + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/http://aims.fao.org/aos/agrovoc/c_13127 + Maracujá + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/http://aims.fao.org/aos/agrovoc/c_13127 + Passion fruit + + + + https://sistemas.sede.embrapa.br/agrotermos/resources/http://aims.fao.org/aos/agrovoc/c_13127 + https://sistemas.sede.embrapa.br/agrotermos/resources/9e2501934d067d71bb1c5850722a73d1 + + + + Acai-verdadeiro + Açaí-verdadeiro + + + + Air-Chilled_chicken + Frango resfriado + + + + Algodao_em_caroco + https://sistemasweb.agricultura.gov.br/sislegis/action/detalhaAto.do?method=visualizarAtoPortalMapa&chave=1830175803#:~:text=3.1.-,Algod%C3%A3o%20em%20caro%C3%A7o%3A%20%C3%A9%20o%20produto%20maduro%20e%20fisiologicamente%20desenvolvido,3.2. + + + + Algodao_em_caroco + Produto maduro e fisiologicamente desenvolvido, oriundo do algodoeiro, que apresenta suas fibras aderidas ao caroço e que ainda não foi beneficiado. + + + + Algodao_em_caroco + Algodão em caroço + + + + Animal_by_product + Animal by-product + + + + Animal_by_product + Subproduto de origem animal + + + + Aquaculture_product + Aquaculture product + + + + Aquaculture_product + Produto da aquicultura + + + + Araucaria + Araucaria + + + + Araucaria + Araucária + + + + Boi_gordo + In the livestock industry, particularly in Brazil, "Boi Gordo" refers to cattle that have been fattened to an optimal weight and condition for slaughter. These animals are typically well-fed and managed to achieve a high level of muscle and fat, making them ideal for meat production. The term signifies the final stage of cattle before they are sent to slaughterhouses, where their market value is assessed based on their weight and overall condition. "Boi Gordo" is a key term in the beef production industry, indicating readiness for processing into meat products. + + + + Boi_gordo + No setor pecuário, especialmente no Brasil, "Boi Gordo" refere-se ao gado que foi engordado até atingir um peso e condição ótimos para o abate. Esses animais são tipicamente bem alimentados e manejados para alcançar um alto nível de músculo e gordura, tornando-os ideais para a produção de carne. O termo significa a etapa final do gado antes de serem enviados para os matadouros, onde seu valor de mercado é avaliado com base em seu peso e condição geral. "Boi Gordo" é um termo-chave na indústria de produção de carne bovina, indicando prontidão para o processamento em produtos de carne. + + + + Boi_gordo + Boi gordo + + + + Boi_gordo + Boi gordo + + + + Boi_gordo + http://dbpedia.org/resource/Fed_cattle + + + + Boi_magro + In the livestock industry, "Boi Magro" refers to cattle that are lean and have not yet reached the optimal weight and condition for slaughter. These animals require additional feeding and management to gain sufficient muscle and fat. + + + + Boi_magro + No setor pecuário, "Boi Magro" refere-se ao gado que é magro e ainda não atingiu o peso e condição ótimos para o abate. Esses animais necessitam de alimentação e manejo adicionais para ganhar músculo e gordura suficientes. + + + + Boi_magro + Boi magro + + + + Buriti + Buriti + + + + Buriti + https://sistemas.sede.embrapa.br/agrotermos/resources/87574e3e9216e89429f8af597b016479 + + + + Buriti + http://aims.fao.org/aos/agrovoc/c_4661 + + + + Buritizeiro + Buritizeiro + + + + By-product + By-product + + + + By-product + Subproduto + + + + By-product + http://aims.fao.org/aos/agrovoc/c_1172 + + + + Canjica_de_milho + Canjica de milho + + + + Carne_ovina + Carne ovina + + + + Cera_de_carnauba + Carnauba wax + + + + Cera_de_carnauba + Cera de carnaúba + + + + Chili_pepper + Chili pepper + + + + Chili_pepper + Pimenta + + + + Citrus_maxima + Citrus maxima + + + + Citrus_maxima + https://www.gbif.org/species/3190160 + + + + Citrus_reticulata + Citrus reticulata + + + + Citrus_reticulata + https://www.gbif.org/species/3190172 + + + + Coco_babacu + Coco de babaçu + + + + Cocoa + Cacau + + + + Cocoa + cocoa + + + + Cocoa + https://sistemas.sede.embrapa.br/agrotermos/resources/b049f571bb7165ea6fb5d377e5dfdfd2 + + + + Coconut + Coco + + + + Coconut + Coconut + + + + Corn_flakes + Corn flakes are made from corn grains that are cooked and then flattened and toasted into thin, crispy flakes. They are widely consumed as breakfast cereals, usually with milk or yogurt, and are also used in some culinary recipes, such as breading or desserts. + + + + Corn_flakes + Flocos de milho, conhecidos popularmente como "cornflakes", são feitos a partir de grãos de milho cozidos e depois esmagados e tostados em flocos finos e crocantes.São amplamente consumidos como cereais matinais, geralmente acompanhados de leite ou iogurte, e também são utilizados em algumas receitas culinárias, como empanados ou sobremesas. + + + + Corn_flakes + Corn flakes + + + + Corn_flakes + Flocos de milho + + + + Cow_cheese + Cow cheese + + + + Cow_cheese + Queijo de vaca + + + + Domestic_chicken + Domestic chicken + + + + Domestic_chicken + Galinha doméstica + + + + Farinha_de_mandioca + Farinha de mandioca + + + + Farinha_de_mandioca + Cassava flour + + + + Farinha_de_mandioca_seca + Farinha de mandioca seca + + + + Farinha_de_mandioca_seca_fina + Farinha de mandioca seca fina + + + + Farinha_de_mandioca_seca_fina + Mandioca seca fina + + + + Farinha_de_mandioca_seca_fina + Farinha de mandioca seca fina + + + + Farinha_de_mandioca_seca_grossa + Farinha de mandioca seca grossa + + + + Farinha_de_mandioca_seca_grossa + Mandioca seca grossa + + + + Farinha_de_mandioca_seca_grossa + Farinha de mandioca seca grossa + + + + Fava_de_anta + Fava-de-anta + + + + Fava_de_anta + fava-d'anta + + + + Fibra + Fibra + + + + Fibra + Vegetal fiber + + + + Fibra + http://aims.fao.org/aos/agrovoc/c_2879 + + + + Fibra + https://sistemas.sede.embrapa.br/agrotermos/resources/aaf228df552d57d8f74534022b863a4f + + + + Fibra_de_carnauba + Fibra de carnaúba + + + + Flocos + Flakes + + + + Flocos + Flocos + + + + Frozen_chicken + Frango congelado + + + + Goat_cheese + Goat cheese + + + + Goat_cheese + Queijo de cabra + + + + Graviola + Graviola + + + + Graviola + https://sistemas.sede.embrapa.br/agrotermos/resources/cd4ad51e3ef278dffda7b240530a6cf7 + + + + Guarana + Guaraná + + + + Guarana + https://sistemas.sede.embrapa.br/agrotermos/resources/da85d12cdbbedf2418030018c66cb0b0 + + + + Jaraqui + Jaraqui + + + + Licurizeiro + Licurizeiro + + + + Licurizeiro + Coqueiro-cabeçudo + + + + Licurizeiro + https://sistemas.sede.embrapa.br/agrotermos/resources/c9ed4cf096ed94bc9c318008ffadbdce + + + + Licurizeiro + Licurizeiro + + + + Mamona_em_baga + Mamona em baga + + + + Massa_de_mandioca + https://periodicos.ufms.br/index.php/EIGEDIN/article/view/7327 + + + + Massa_de_mandioca + Massa de mandioca + + + + Maxixe + Maxixe + + + + Maxixe + https://sistemas.sede.embrapa.br/agrotermos/resources/2ba40885c990c681fbb1872fc59a5a16 + + + + Murumuru + Murumuru + + + + Murumuzeiro + Palmeira murumuru + + + + OWLClass_59217d1a_950e_4f2d_b80c_695e0fe9100dalm + Hive product + + + + OWLClass_59217d1a_950e_4f2d_b80c_695e0fe9100dalm + Produto da apicultura + + + + OWLClass_76cbdf78_9da4_43ce_b015_50ee8259fab6alm + Livestock product + + + + OWLClass_76cbdf78_9da4_43ce_b015_50ee8259fab6alm + Produto pecuário + + + + OWLClass_a7c4f0d2_6654_4dc6_ae34_97ce99b35adealm + Produto da vida selvagem + + + + OWLClass_a7c4f0d2_6654_4dc6_ae34_97ce99b35adealm + Wildlife product + + + + Oleo_de_buriti + Óleo de buriti + + + + Oleo_de_copaiba + Óleo de copaíba + + + + Oleo_de_copaiba + http://aims.fao.org/aos/agrovoc/c_10091 + + + + Oleo_de_copaiba + https://sistemas.sede.embrapa.br/agrotermos/resources/29fb6a538d8ed7225c2787a2118752aa + + + + Oleo_de_murumuru + Murumuru oil + + + + Oleo_de_murumuru + Óleo de murumuru + + + + Oleo_de_pequi + Óleo de pequi + + + + Organism + Organism + + + + Organism + Organismo + + + + Organism + http://purl.obolibrary.org/obo/OBI_0100026 + + + + Ovo_de_galinha + Ovo de galinha + + + + Pepper + Pepper + + + + Pepper + Pimenta do Reino + + + + Pepper + Black pepper + + + + Pinus + Pinus + + + + Pinus + Pinus + + + + Plant_by-product + Plant by-product + + + + Plant_by-product + Subproduto de origem vegetal + + + + Planta_produtora_de_celulose + Cellulose-producing plant + + + + Planta_produtora_de_celulose + Planta produtora de celulose + + + + Po_cerifero + Pó cerífero + + + + Po_cerifero_de_carnauba + Pó cerífero de carnaúba + + + + Polpa_de_abacaxi + Polpa de abacaxi + + + + Polpa_de_acai + Assai pulp + + + + Polpa_de_acai + Polpa de açaí + + + + Polpa_de_acerola + Polpa de acerola + + + + Polpa_de_tamarindo + Polpa de tamarindo + + + + Polvilho + Polvilho + + + + Polvilho + Fécula de mandioca + + + + Polvilho + Polvilho + + + + Poultry_product + Poultry product + + + + Poultry_product + Produto da avicultura + + + + Processed_animal_product + Processed animal product + + + + Processed_animal_product + Produto de origem animal processado + + + + Product_type + Product type + + + + Product_type + Tipo de produto + + + + Raw_animal_product + Produto de origem animal in natura + + + + Raw_animal_product + Raw animal product + + + + Raw_plant_product + Produto de origem vegetal in natura + + + + Raw_plant_product + Raw plant product + + + + Raw_product + Produto in natura + + + + Raw_product + Raw product + + + + Raw_product + http://aims.fao.org/aos/agrovoc/c_24887 + + + + Rice_flakes + Flocos de arroz são grãos de arroz que foram processados para se tornarem finos e achatados. Este processo envolve cozinhar os grãos, desidratá-los e prensá-los, resultando em flocos leves e crocantes. Eles são comumente usados em cereais matinais, barras de cereais e podem ser consumidos com leite, iogurte ou utilizados em receitas como bolos e biscoitos. + + + + Rice_flakes + Rice flakes are grains of rice that have been processed to become thin and flattened. This process involves cooking the grains, dehydrating them, and pressing them, resulting in light and crispy flakes. They are commonly used in breakfast cereals, cereal bars, and can be eaten with milk, yogurt, or used in recipes like cakes and cookies. + + + + Rice_flakes + Flocos de arrroz + + + + Rice_flakes + Rice flakes + + + + Sericulture_product + Produto da sericicultura + + + + Sericulture_product + Sericulture product + + + + Sericulture_product + http://aims.fao.org/aos/agrovoc/c_6981 + + + + Seringueira + Seringueira + + + + Seringueira + Sharinga tree + + + + Soybean_meal + Farelo de soja + + + + Soybean_meal + Soybean meal + + + + Sucroenergetico + Sucroenergético + + + + Sugar_plant + Planta produtora de açúcar + + + + Sugar_plant + Sugar-producing plant + + + + Tamarindo + Tamarind + + + + Tamarindo + Tamarindo + + + + Tamarindo + https://sistemas.sede.embrapa.br/agrotermos/resources/5bf4ec8db1d13c79edfae0699ce662fe + + + + Tomato_extract + Extrato de tomate + + + + Tomato_extract + Tomato extract + + + + Triticum + Triticum + + + + Triticum + https://www.gbif.org/species/2706388 + + + + algodaoPluma + https://sistemasweb.agricultura.gov.br/sislegis/action/detalhaAto.do?method=visualizarAtoPortalMapa&chave=1830175803#:~:text=3.1.-,Algod%C3%A3o%20em%20caro%C3%A7o%3A%20%C3%A9%20o%20produto%20maduro%20e%20fisiologicamente%20desenvolvido,3.2. + + + + algodaoPluma + Produto resultante da operação de beneficiamento do algodão em caroço. + + + + algodaoPluma + Algodão em pluma + + + + amendoaandiroba + Amêndoa de andiroba + + + + amendoaandiroba + https://sistemas.sede.embrapa.br/agrotermos/resources/8d11f918a65e21f7695af788b28001d5 + + + + amendoabaru + Amêndoa de cumbaru + + + + amendoabaru + Amêndoa de baru + + + + amendoacacau + Amêndoa de cacau + + + + amendoacacau + Cocoa almond + + + + amendoacacau + https://sistemas.sede.embrapa.br/agrotermos/resources/b049f571bb7165ea6fb5d377e5dfdfd2 + + + + amendoadeouricuri + Amêndoa de licuri + + + + amendoadeouricuri + Licuri + + + + amendoadeouricuri + Ouricuri + + + + amendoamacauba + Amêndoa de macaúba + + + + amendoamacauba + Macaúba + + + + amendoamacauba + Amêndoa de macaúba + + + + azeitebabacu + Azeite de babaçu + + + + azeitebabacu + Óleo de babaçu + + + + azeitebabacu + Azeite de babaçu + + + + azeitemacauba + Azeite de macaúba + + + + azeitemacauba + https://sistemas.sede.embrapa.br/agrotermos/resources/513d52ec57a549910bf9ced35be817a8 + + + + bebibalactea + Bebida láctea + + + + bebibalactea + Dairy drink + + + + cheiroverde + Cheiro verde + + + + etanolanidro + Etanol anidro + + + + etanolhidratado + Etanol hidratado + + + + fitweed + Chicória-do-Pará + + + + fitweed + Fitweed + + + + fitweed + Coentro-bravo + + + + has_ingredient + Esta propriedade indica que um produto contém um ou mais ingredientes específicos. + + + + has_ingredient + This property indicates that a product contains one or more specific ingredients. + + + + has_ingredient + contém ingrediente + + + + has_ingredient + has ingredient + + + + is_a_hybrid_of + Esta propriedade indica que uma dada espécie é um híbrido de outras duas espécies. + + + + is_a_hybrid_of + This property indicates that a given species is a hybrid of two other species. + + + + is_a_hybrid_of + is a hybrid of + + + + is_a_hybrid_of + é um híbrido de + + + + is_ingredient_of + Esta propriedade indica que algo é um ingrediente de um produto específico. + + + + is_ingredient_of + This property indicates that something is an ingredient of a specific product. + + + + is_ingredient_of + is ingredient of + + + + is_ingredient_of + é ingrediente de + + + + polpaburiti + Polpa de buriti + + + + polpacacau + Polpa de cacau + + + + polpacaja + Polpa de cajá + + + + polpacaju + Polpa de caju + + + + polpaceriguela + Polpa de ceriguela + + + + polpaceriguela + Polpa de seriguela + + + + polpaceriguela + Polpa de ceriguela + + + + polpacupuacu + Polpa de cupuaçu + + + + polpacupuacu + https://sistemas.sede.embrapa.br/agrotermos/resources/93091ce029083f6c99e904493ae41517 + + + + polpagoiaba + Polpa de goiaba + + + + polpagraviola + Polpa de graviola + + + + polpajenipapo + Polpa de jenipapo + + + + polpamanga + Polpa de manga + + + + polpamangaba + Polpa de mangaba + + + + polpamaracuja + Polpa de maracujá + + + + polpaumbu + Polpa de umbu + + + + residue_of + A propriedade "residue_of" liga um subproduto ao produto cuja produção resulta nesse subproduto. Essa relação indica que o subproduto é um resíduo gerado durante o processo de produção do produto principal. + + + + residue_of + The "residue_of" property links a by-product to the product whose production results in this by-product. This relationship indicates that the by-product is a residue generated during the manufacturing process of the main product. + + + + residue_of + residue of + + + + residue_of + resíduo da produção de + + + + suinoVivo + Suíno vivo + + + + trigomelhorador + Trigo melhorador + + + + vacaGorda + Vaca gorda + + + + https://www.gbif.org/species/1 + Animalia + + + + https://www.gbif.org/species/1069 + Osteoglossiformes + + + + https://www.gbif.org/species/10779983 + Dipteryx odorata + + + + https://www.gbif.org/species/10792217 + Ilex + + + + https://www.gbif.org/species/1169 + Asparagales + + + + https://www.gbif.org/species/1176 + Solanales + + + + https://www.gbif.org/species/1334757 + Apis + + + + https://www.gbif.org/species/1341976 + Apis mellifera + + + + https://www.gbif.org/species/1351 + Apiales + + + + https://www.gbif.org/species/1353 + Ericales + + + + https://www.gbif.org/species/1354 + Fagales + + + + https://www.gbif.org/species/1369 + Poales + + + + https://www.gbif.org/species/1370 + Fabales + + + + https://www.gbif.org/species/1414 + Malpighiales + + + + https://www.gbif.org/species/1457 + Hymenoptera + + + + https://www.gbif.org/species/1868660 + Bombyx + + + + https://www.gbif.org/species/1868664 + Bombyx mori + + + + https://www.gbif.org/species/194 + Pinopsida + + + + https://www.gbif.org/species/196 + Liliopsida + + + + https://www.gbif.org/species/212 + Aves + + + + https://www.gbif.org/species/216 + Insecta + + + + https://www.gbif.org/species/220 + Magnoliopsida + + + + https://www.gbif.org/species/2352125 + Semaprochilodus + + + + https://www.gbif.org/species/2352126 + Semaprochilodus insignis + + + + https://www.gbif.org/species/2352134 + Semaprochilodus taeniurus + + + + https://www.gbif.org/species/2352148 + Prochilodus + + + + https://www.gbif.org/species/2352151 + Prochilodus brevis + + + + https://www.gbif.org/species/2352154 + Prochilodus lineatus + + + + https://www.gbif.org/species/2352177 + Prochilodus argenteus + + + + https://www.gbif.org/species/2353218 + Piaractus + + + + https://www.gbif.org/species/2353219 + Piaractus mesopotamicus + + + + https://www.gbif.org/species/2370578 + Tilapia + + + + https://www.gbif.org/species/2378 + Passifloraceae + + + + https://www.gbif.org/species/2389 + Convolvulaceae + + + + https://www.gbif.org/species/2396 + Rutaceae + + + + https://www.gbif.org/species/2397 + Meliaceae + + + + https://www.gbif.org/species/2398 + Anacardiaceae + + + + https://www.gbif.org/species/2402332 + Arapaima + + + + https://www.gbif.org/species/2441017 + Bos + + + + https://www.gbif.org/species/2441022 + Bos taurus + + + + https://www.gbif.org/species/2441047 + Capra + + + + https://www.gbif.org/species/2441056 + Capra hircus + + + + https://www.gbif.org/species/2441110 + Ovis aries + + + + https://www.gbif.org/species/2441213 + Sus + + + + https://www.gbif.org/species/2473720 + Gallus + + + + https://www.gbif.org/species/2499 + Juglandaceae + + + + https://www.gbif.org/species/2684241 + Pinus + + + + https://www.gbif.org/species/2684910 + Araucaria + + + + https://www.gbif.org/species/2684940 + Araucaria angustifolia + + + + https://www.gbif.org/species/2699430 + Ananas + + + + https://www.gbif.org/species/2703201 + Cenchrus + + + + https://www.gbif.org/species/2703325 + A Man-made cereal grass crop obtained from hybridization of wheat (Triticum spp) with rye (Secale cereale). + + + + https://www.gbif.org/species/2703325 + https://doi.org/10.1007/978-0-387-72297-9_9 + + + + https://www.gbif.org/species/2703325 + ×Triticosecale + + + + https://www.gbif.org/species/2703455 + Oryza + + + + https://www.gbif.org/species/2703459 + Oryza sativa + + + + https://www.gbif.org/species/2703464 + Oryza glaberrima + + + + https://www.gbif.org/species/2703910 + Saccharum + + + + https://www.gbif.org/species/2705049 + Zea + + + + https://www.gbif.org/species/2705180 + Sorghum + + + + https://www.gbif.org/species/2705181 + Sorghum bicolor + + + + https://www.gbif.org/species/2705282 + Avena + + + + https://www.gbif.org/species/2705290 + Avena sativa + + + + https://www.gbif.org/species/2705292 + Avena byzantina + + + + https://www.gbif.org/species/2705965 + Secale + + + + https://www.gbif.org/species/2705966 + Secale cereale + + + + https://www.gbif.org/species/2706050 + Hordeum + + + + https://www.gbif.org/species/2706056 + Hordeum vulgare + + + + https://www.gbif.org/species/2706141 + Pennisetum glaucum + + + + https://www.gbif.org/species/2732585 + Leopoldinia + + + + https://www.gbif.org/species/2732586 + Leopoldinia piassaba + + + + https://www.gbif.org/species/2732664 + Attalea + + + + https://www.gbif.org/species/2732833 + Attalea funifera + + + + https://www.gbif.org/species/2732877 + Attalea speciosa + + + + https://www.gbif.org/species/2735116 + Cocos + + + + https://www.gbif.org/species/2735117 + Cocos nucifera + + + + https://www.gbif.org/species/2737041 + Syagrus + + + + https://www.gbif.org/species/2738048 + Astrocaryum + + + + https://www.gbif.org/species/2738081 + Astrocaryum murumuru + + + + https://www.gbif.org/species/2738233 + Copernicia + + + + https://www.gbif.org/species/2738262 + Copernicia prunifera + + + + https://www.gbif.org/species/2739147 + Acrocomia + + + + https://www.gbif.org/species/2760990 + Musa + + + + https://www.gbif.org/species/2762680 + Musa acuminata + + + + https://www.gbif.org/species/2762950 + Musa balbisiana + + + + https://www.gbif.org/species/2766430 + Agave + + + + https://www.gbif.org/species/2855860 + Allium schoenoprasum + + + + https://www.gbif.org/species/2856681 + Allium sativum + + + + https://www.gbif.org/species/2857697 + Allium cepa + + + + https://www.gbif.org/species/2874172 + Passiflora + + + + https://www.gbif.org/species/2874190 + Passiflora edulis + + + + https://www.gbif.org/species/2874482 + Carica + + + + https://www.gbif.org/species/2874484 + Carica papaya + + + + https://www.gbif.org/species/2874506 + Cucurbita + + + + https://www.gbif.org/species/2874515 + Cucurbita maxima + + + + https://www.gbif.org/species/2874568 + Cucumis + + + + https://www.gbif.org/species/2874569 + Cucumis sativus + + + + https://www.gbif.org/species/2874570 + Cucumis melo + + + + https://www.gbif.org/species/2874573 + Cucumis anguria + + + + https://www.gbif.org/species/2874621 + Citrullus lanatus + + + + https://www.gbif.org/species/2895315 + Coffea + + + + https://www.gbif.org/species/2895345 + Coffea arabica + + + + https://www.gbif.org/species/2895528 + Coffea canephora + + + + https://www.gbif.org/species/2895591 + Genipa + + + + https://www.gbif.org/species/2895593 + Genipa americana + + + + https://www.gbif.org/species/2928509 + Ipomoea + + + + https://www.gbif.org/species/2928551 + Ipomoea batatas + + + + https://www.gbif.org/species/2928756 + Nicotiana + + + + https://www.gbif.org/species/2928774 + Nicotiana tabacum + + + + https://www.gbif.org/species/2928997 + Solanum + + + + https://www.gbif.org/species/2930137 + Solanum lycopersicum + + + + https://www.gbif.org/species/2930262 + Solanum tuberosum + + + + https://www.gbif.org/species/2930617 + Solanum melongena + + + + https://www.gbif.org/species/2932937 + Capsicum + + + + https://www.gbif.org/species/2932944 + Capsicum annuum + + + + https://www.gbif.org/species/2947121 + Dipteryx + + + + https://www.gbif.org/species/2947798 + Phaseolus + + + + https://www.gbif.org/species/2956684 + Arachis + + + + https://www.gbif.org/species/2974262 + Dimorphandra + + + + https://www.gbif.org/species/2974269 + Dimorphandra mollis + + + + https://www.gbif.org/species/2974751 + Vicia + + + + https://www.gbif.org/species/2974832 + Vicia faba + + + + https://www.gbif.org/species/2975767 + Tamarindus + + + + https://www.gbif.org/species/2975768 + Tamarindus indica + + + + https://www.gbif.org/species/2978115 + Copaifera + + + + https://www.gbif.org/species/2978132 + Copaifera multijuga + + + + https://www.gbif.org/species/2978171 + Copaifera langsdorffii + + + + https://www.gbif.org/species/2984934 + Theobroma + + + + https://www.gbif.org/species/3001068 + Malus + + + + https://www.gbif.org/species/3001244 + Malus domestica + + + + https://www.gbif.org/species/3020559 + Prunus + + + + https://www.gbif.org/species/3032212 + Corchorus + + + + https://www.gbif.org/species/3034387 + Eryngium + + + + https://www.gbif.org/species/3034425 + Eryngium foetidum + + + + https://www.gbif.org/species/3034740 + Daucus + + + + https://www.gbif.org/species/3034742 + Daucus carota + + + + https://www.gbif.org/species/3034870 + Coriandrum + + + + https://www.gbif.org/species/3034871 + Coriandrum sativum + + + + https://www.gbif.org/species/3034906 + Petroselinum + + + + https://www.gbif.org/species/3042506 + Brassica + + + + https://www.gbif.org/species/3042636 + Brassica napus + + + + https://www.gbif.org/species/3042845 + Brassica oleracea + + + + https://www.gbif.org/species/3049319 + Eruca + + + + https://www.gbif.org/species/3054350 + Juglans + + + + https://www.gbif.org/species/3054368 + Juglans regia + + + + https://www.gbif.org/species/3060685 + Manihot + + + + https://www.gbif.org/species/3060998 + Manihot esculenta + + + + https://www.gbif.org/species/3065 + Asteraceae + + + + https://www.gbif.org/species/3071157 + Hevea + + + + https://www.gbif.org/species/3071171 + Hevea brasiliensis + + + + https://www.gbif.org/species/3073 + Poaceae + + + + https://www.gbif.org/species/3075433 + Piper + + + + https://www.gbif.org/species/3075453 + Mangifera + + + + https://www.gbif.org/species/3083179 + Bertholletia + + + + https://www.gbif.org/species/3086357 + Piper nigrum + + + + https://www.gbif.org/species/3112 + Brassicaceae + + + + https://www.gbif.org/species/3119134 + Helianthus + + + + https://www.gbif.org/species/3140231 + Lactuca + + + + https://www.gbif.org/species/3152082 + Corchorus capsularis + + + + https://www.gbif.org/species/3152205 + Theobroma cacao + + + + https://www.gbif.org/species/3152208 + Theobroma grandiflorum + + + + https://www.gbif.org/species/3152652 + Gossypium + + + + https://www.gbif.org/species/3152705 + Abelmoschus + + + + https://www.gbif.org/species/3152707 + Abelmoschus esculentus + + + + https://www.gbif.org/species/3155252 + Annona + + + + https://www.gbif.org/species/3169678 + Hancornia + + + + https://www.gbif.org/species/3169679 + Hancornia speciosa + + + + https://www.gbif.org/species/3187232 + Psidium + + + + https://www.gbif.org/species/3189661 + Caryocar + + + + https://www.gbif.org/species/3189663 + Caryocar brasiliense + + + + https://www.gbif.org/species/3189948 + Paullinia + + + + https://www.gbif.org/species/3189949 + Paullinia cupana + + + + https://www.gbif.org/species/3190155 + Citrus + + + + https://www.gbif.org/species/3190164 + Citrus aurantiifolia + + + + https://www.gbif.org/species/3190512 + Carapa + + + + https://www.gbif.org/species/3190513 + Carapa guianensis + + + + https://www.gbif.org/species/3190592 + Spondias + + + + https://www.gbif.org/species/3190598 + Spondias purpurea + + + + https://www.gbif.org/species/3190638 + Mangifera indica + + + + https://www.gbif.org/species/3191274 + Malpighia + + + + https://www.gbif.org/species/3191347 + Byrsonima + + + + https://www.gbif.org/species/3191361 + Byrsonima crassifolia + + + + https://www.gbif.org/species/359 + Mammalia + + + + https://www.gbif.org/species/3740 + Bromeliaceae + + + + https://www.gbif.org/species/3924 + Araucariaceae + + + + https://www.gbif.org/species/3925 + Pinaceae + + + + https://www.gbif.org/species/3990 + Lecythidaceae + + + + https://www.gbif.org/species/404 + Piperales + + + + https://www.gbif.org/species/412 + Gentianales + + + + https://www.gbif.org/species/414 + Asterales + + + + https://www.gbif.org/species/4228 + Arapaimidae + + + + https://www.gbif.org/species/4287106 + Serrasalmidae + + + + https://www.gbif.org/species/4334 + Apidae + + + + https://www.gbif.org/species/44 + Chordata + + + + https://www.gbif.org/species/4686 + Musaceae + + + + https://www.gbif.org/species/4691 + Euphorbiaceae + + + + https://www.gbif.org/species/5014 + Myrtaceae + + + + https://www.gbif.org/species/5015 + Rosaceae + + + + https://www.gbif.org/species/5212877 + Arapaima gigas + + + + https://www.gbif.org/species/5288819 + Ananas comosus + + + + https://www.gbif.org/species/5290052 + Zea mays + + + + https://www.gbif.org/species/5293398 + Euterpe oleracea + + + + https://www.gbif.org/species/5293403 + Euterpe edulis + + + + https://www.gbif.org/species/5293875 + Syagrus coronata + + + + https://www.gbif.org/species/5294777 + Mauritia flexuosa + + + + https://www.gbif.org/species/5302 + Suidae + + + + https://www.gbif.org/species/5330758 + Colocasia + + + + https://www.gbif.org/species/5330776 + Colocasia esculenta + + + + https://www.gbif.org/species/5350452 + Phaseolus vulgaris + + + + https://www.gbif.org/species/5353770 + Arachis hypogaea + + + + https://www.gbif.org/species/5359660 + Glycine max + + + + https://www.gbif.org/species/537 + Characiformes + + + + https://www.gbif.org/species/5372392 + Vitis vinifera + + + + https://www.gbif.org/species/5380041 + Ricinus communis + + + + https://www.gbif.org/species/5386 + Fabaceae + + + + https://www.gbif.org/species/54 + Arthropoda + + + + https://www.gbif.org/species/5407099 + Annona squamosa + + + + https://www.gbif.org/species/5407273 + Annona muricata + + + + https://www.gbif.org/species/5414252 + Ilex paraguariensis + + + + https://www.gbif.org/species/5420380 + Psidium guajava + + + + https://www.gbif.org/species/5421367 + Anacardium + + + + https://www.gbif.org/species/5421368 + Anacardium occidentale + + + + https://www.gbif.org/species/5421429 + Malpighia glabra + + + + https://www.gbif.org/species/551 + Alismatales + + + + https://www.gbif.org/species/552 + Arecales + + + + https://www.gbif.org/species/5828197 + Cenchrus americanus + + + + https://www.gbif.org/species/587 + Perciformes + + + + https://www.gbif.org/species/6 + Plantae + + + + https://www.gbif.org/species/627 + Zingiberales + + + + https://www.gbif.org/species/640 + Pinales + + + + https://www.gbif.org/species/6634 + Cucurbitaceae + + + + https://www.gbif.org/species/6636 + Caricaceae + + + + https://www.gbif.org/species/6647 + Caryocaraceae + + + + https://www.gbif.org/species/6657 + Sapindaceae + + + + https://www.gbif.org/species/6672 + Vitaceae + + + + https://www.gbif.org/species/6676 + Malpighiaceae + + + + https://www.gbif.org/species/6678 + Piperaceae + + + + https://www.gbif.org/species/6685 + Malvaceae + + + + https://www.gbif.org/species/6701 + Apocynaceae + + + + https://www.gbif.org/species/6716 + Aquifoliaceae + + + + https://www.gbif.org/species/6720 + Apiaceae + + + + https://www.gbif.org/species/690 + Myrtales + + + + https://www.gbif.org/species/691 + Rosales + + + + https://www.gbif.org/species/6979 + Araceae + + + + https://www.gbif.org/species/718 + Magnoliales + + + + https://www.gbif.org/species/7222050 + Vitales + + + + https://www.gbif.org/species/7224005 + Cucurbitales + + + + https://www.gbif.org/species/7225535 + Brassicales + + + + https://www.gbif.org/species/7226638 + Aquifoliales + + + + https://www.gbif.org/species/723 + Galliformes + + + + https://www.gbif.org/species/731 + Artiodactyla + + + + https://www.gbif.org/species/7331 + Prochilodontidae + + + + https://www.gbif.org/species/7403263 + Lactuca sativa + + + + https://www.gbif.org/species/7413879 + Acrocomia aculeata + + + + https://www.gbif.org/species/7467468 + Vitis + + + + https://www.gbif.org/species/7474861 + Eruca sativa + + + + https://www.gbif.org/species/7493935 + Eucalyptus + + + + https://www.gbif.org/species/7647136 + Citrus limon + + + + https://www.gbif.org/species/7681 + Arecaceae + + + + https://www.gbif.org/species/7682 + Amaryllidaceae + + + + https://www.gbif.org/species/7683 + Asparagaceae + + + + https://www.gbif.org/species/7707728 + Tracheophyta + + + + https://www.gbif.org/species/7717 + Solanaceae + + + + https://www.gbif.org/species/7828157 + Petroselinum crispum + + + + https://www.gbif.org/species/7906352 + Euterpe + + + + https://www.gbif.org/species/7968287 + Glycine + + + + https://www.gbif.org/species/797 + Lepidoptera + + + + https://www.gbif.org/species/8077391 + Citrus aurantium + + + + https://www.gbif.org/species/8088560 + Bertholletia excelsa + + + + https://www.gbif.org/species/8123118 + Agave sisalana + + + + https://www.gbif.org/species/8149923 + Prunus persica + + + + https://www.gbif.org/species/8185571 + Spondias mombin + + + + https://www.gbif.org/species/8230360 + Spondias tuberosa + + + + https://www.gbif.org/species/8257011 + Citrullus + + + + https://www.gbif.org/species/8413640 + Mauritia + + + + https://www.gbif.org/species/8522 + Cichlidae + + + + https://www.gbif.org/species/8717303 + Cucurbita pepo + + + + https://www.gbif.org/species/8798 + Rubiaceae + + + + https://www.gbif.org/species/8832 + Bombycidae + + + + https://www.gbif.org/species/9188216 + Ricinus + + + + https://www.gbif.org/species/9206251 + Helianthus annuus + + + + https://www.gbif.org/species/9291 + Annonaceae + + + + https://www.gbif.org/species/9326020 + Gallus gallus + + + + https://www.gbif.org/species/933 + Sapindales + + + + https://www.gbif.org/species/9331 + Phasianidae + + + + https://www.gbif.org/species/941 + Malvales + + + + https://www.gbif.org/species/9457155 + Gallus gallus f. domesticus + + + + https://www.gbif.org/species/9531221 + Ovis + + + + https://www.gbif.org/species/9614 + Bovidae + + + + https://www.gbif.org/species/9624496 + Allium + + + + + + + + + + + + + + + + xsd:anyURI + + + + xsd:anyURI + + + + xsd:anyURI + + + + xsd:anyURI + + + + xsd:anyURI + + + + + + diff --git a/test/docker_infrastructure.rb b/test/docker_infrastructure.rb index 0c81d7b1f..46c2b4a76 100644 --- a/test/docker_infrastructure.rb +++ b/test/docker_infrastructure.rb @@ -51,7 +51,7 @@ def docker_tests require_relative "test_case" test_files.each {|f| require f} - MiniTest::Unit.runner.run + Minitest.run rescue => e puts e.message puts e.backtrace.join("\n\t") @@ -95,7 +95,7 @@ def docker_tests_forked(forks) test_files_sliced[i].each {|f| require f} - MiniTest::Unit.runner.run + Minitest.run rescue => e sio << "\n#{e.message}\n#{e.backtrace.join("\t\n")}" ensure @@ -119,4 +119,3 @@ def docker_tests_forked(forks) Kernel.exit! # force the process to quit without minitest's autorun (triggered on at_exit) end end - diff --git a/test/http_cache/test_http_cache.rb b/test/http_cache/test_http_cache.rb index 782ea72b5..c6e9b9e3a 100644 --- a/test/http_cache/test_http_cache.rb +++ b/test/http_cache/test_http_cache.rb @@ -24,7 +24,7 @@ def self.after_suite def _ontology_and_class results = create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) ontology = results[2].first - cls = LinkedData::Models::Class.where.include(:prefLabel).in(ontology.latest_submission).page(1, 1).first + cls = LinkedData::Models::Class.where.include(:prefLabel).in(ontology.latest_submission).first return ontology, cls end @@ -57,6 +57,7 @@ def test_cache_segment_proper_setup end def test_cache_segment_invalidate + create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) classes = LinkedData::Models::Class.where.include(:prefLabel).in(@@ontology.latest_submission).page(1, 100).to_a classes.each {|c| c.cache_write} last_modified_values = classes.map {|c| c.last_modified} @@ -67,6 +68,7 @@ def test_cache_segment_invalidate end def test_cache_segment_invalidate_when_member_invalidates + create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: true) classes = LinkedData::Models::Class.where.include(:prefLabel).in(@@ontology.latest_submission).page(1, 100).to_a classes.each {|c| c.cache_write} last_modified_values = classes.map {|c| c.last_modified} @@ -137,4 +139,4 @@ def test_cache_invalidate_all assert_equal [], LinkedData::HTTPCache.keys_for_invalidate_all end -end \ No newline at end of file +end diff --git a/test/models/skos/test_collections.rb b/test/models/skos/test_collections.rb index c2c1c3d07..485c7ad53 100644 --- a/test/models/skos/test_collections.rb +++ b/test/models/skos/test_collections.rb @@ -18,11 +18,11 @@ def test_collections_all collections = LinkedData::Models::SKOS::Collection.in(sub).include(:members, :prefLabel).all assert_equal 2, collections.size - collections_test = test_data + collections_test = test_data.to_h { |x| [x[:id], x] } - collections.each_with_index do |x, i| - collection_test = collections_test[i] - assert_equal collection_test[:id], x.id.to_s + collections.each do |x| + collection_test = collections_test[x.id.to_s] + refute_nil collection_test assert_equal collection_test[:prefLabel], x.prefLabel assert_equal collection_test[:memberCount], x.memberCount end diff --git a/test/models/skos/test_schemes.rb b/test/models/skos/test_schemes.rb index ca1bf686c..c66c87384 100644 --- a/test/models/skos/test_schemes.rb +++ b/test/models/skos/test_schemes.rb @@ -25,7 +25,11 @@ def test_schemes_all schemes.each_with_index do |x, i| scheme_test = schemes_test[i] assert_equal scheme_test[:id], x.id.to_s - assert_equal scheme_test[:prefLabel], x.prefLabel + if scheme_test[:prefLabel].nil? + assert_nil x.prefLabel + else + assert_equal scheme_test[:prefLabel], x.prefLabel + end end end diff --git a/test/models/test_class.rb b/test/models/test_class.rb index d5d70cd4b..8ce920c32 100644 --- a/test/models/test_class.rb +++ b/test/models/test_class.rb @@ -7,7 +7,7 @@ def test_class_parents acr = "CSTPROPS" init_test_ontology_msotest acr - os = LinkedData::Models::OntologySubmission.where(ontology: [ acronym: acr ], + os = LinkedData::Models::OntologySubmission.where(ontology: [acronym: acr], submissionId: 1).all assert(os.length == 1) os = os[0] @@ -16,35 +16,34 @@ def test_class_parents cls = LinkedData::Models::Class.find(class_id).in(os).include(:parents).to_a[0] pp = cls.parents[0] - assert_equal(os.id,pp.submission.id) + assert_equal(os.id, pp.submission.id) pp.bring(:parents) assert pp.parents.length == 1 assert_equal(os.id, pp.parents.first.submission.id) - #read_only + # read_only cls = LinkedData::Models::Class.find(class_id).in(os).include(:parents).read_only.all[0] pp = cls.parents[0] - assert_equal(os.id,pp.submission.id) + assert_equal(os.id, pp.submission.id) class_id = RDF::IRI.new "http://bioportal.bioontology.org/ontologies/msotes#class_5" cls = LinkedData::Models::Class.find(class_id).in(os).include(:parents).first parents = cls.parents assert_equal(parents, cls.parents) assert_equal(3, cls.parents.length) - parent_ids = [ "http://bioportal.bioontology.org/ontologies/msotes#class2", - "http://bioportal.bioontology.org/ontologies/msotes#class4", - "http://bioportal.bioontology.org/ontologies/msotes#class3" ] + parent_ids = ["http://bioportal.bioontology.org/ontologies/msotes#class2", + "http://bioportal.bioontology.org/ontologies/msotes#class4", + "http://bioportal.bioontology.org/ontologies/msotes#class3"] parent_id_db = cls.parents.map { |x| x.id.to_s } assert_equal(parent_id_db.sort, parent_ids.sort) assert !cls.parents[0].submission.nil? - #they should have the same submission + # they should have the same submission assert_equal(cls.parents[0].submission, os) - #transitive - assert_raises ArgumentError do - cls.bring(:ancestors) - end + # transitive + cls.bring(:ancestors) + assert_includes cls.loaded_attributes.to_a, :ancestors ancestors = cls.ancestors.dup ancestors.each do |a| assert !a.submission.nil? @@ -52,9 +51,9 @@ def test_class_parents assert ancestors.length == cls.ancestors.length ancestors.map! { |a| a.id.to_s } data_ancestors = ["http://bioportal.bioontology.org/ontologies/msotes#class1", - "http://bioportal.bioontology.org/ontologies/msotes#class2", - "http://bioportal.bioontology.org/ontologies/msotes#class4", - "http://bioportal.bioontology.org/ontologies/msotes#class3"] + "http://bioportal.bioontology.org/ontologies/msotes#class2", + "http://bioportal.bioontology.org/ontologies/msotes#class4", + "http://bioportal.bioontology.org/ontologies/msotes#class3"] assert ancestors.sort == data_ancestors.sort end @@ -63,7 +62,7 @@ def test_class_children acr = "CSTPROPS" init_test_ontology_msotest acr - os = LinkedData::Models::OntologySubmission.where(ontology: [ acronym: acr ], + os = LinkedData::Models::OntologySubmission.where(ontology: [acronym: acr], submissionId: 1).all assert(os.length == 1) os = os[0] @@ -71,29 +70,27 @@ def test_class_children class_id = RDF::IRI.new "http://bioportal.bioontology.org/ontologies/msotes#class1" cls = LinkedData::Models::Class.find(class_id).in(os) - .include(:parents) - .include(:children) - .to_a[0] + .include(:parents) + .include(:children) + .to_a[0] children = cls.children assert_equal(1, cls.children.length) children_id = "http://bioportal.bioontology.org/ontologies/msotes#class2" - assert_equal(children_id,cls.children[0].id.to_s) + assert_equal(children_id, cls.children[0].id.to_s) - #they should have the same submission + # they should have the same submission assert_equal(cls.children[0].submission, os) - #transitive - assert_raises ArgumentError do - cls.bring(:descendants) - end + # transitive + cls.bring(:descendants) + assert_includes cls.loaded_attributes.to_a, :descendants descendants = cls.descendants.dup descendants.map! { |a| a.id.to_s } data_descendants = ["http://bioportal.bioontology.org/ontologies/msotes#class_5", - "http://bioportal.bioontology.org/ontologies/msotes#class2", - "http://bioportal.bioontology.org/ontologies/msotes#class_7"] + "http://bioportal.bioontology.org/ontologies/msotes#class2", + "http://bioportal.bioontology.org/ontologies/msotes#class_7"] assert descendants.sort == data_descendants.sort - - page = cls.retrieve_descendants(page=2,size=2) + page = cls.retrieve_descendants(page = 2, size = 2) assert page.total_pages == 2 assert page.prev_page == 1 assert page.next_page == nil @@ -101,16 +98,16 @@ def test_class_children assert page[0].id.to_s == data_descendants[2] cls = LinkedData::Models::Class.find(class_id).in(os) - .to_a[0] + .to_a[0] cls.load_has_children has_c = cls.hasChildren - assert_equal(has_c,true) + assert_equal(has_c, true) class_id = RDF::IRI.new "http://bioportal.bioontology.org/ontologies/msotes#class_7" cls = LinkedData::Models::Class.find(class_id).in(os) - .to_a[0] + .to_a[0] cls.load_has_children has_c = cls.hasChildren - assert_equal(has_c,false) + assert_equal(has_c, false) end def test_path_to_root diff --git a/test/models/test_class_request_lang.rb b/test/models/test_class_request_lang.rb index ac8d57a85..84ea6841e 100644 --- a/test/models/test_class_request_lang.rb +++ b/test/models/test_class_request_lang.rb @@ -21,15 +21,32 @@ def self.parse process_rdf: true, index_search: false, run_metrics: false, reasoning: false ) + new('').submission_parse('APTO', 'Test parse ontology has iri label', + 'test/data/ontology_files/apto.owl', 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false + ) end def teardown reset_lang end + def test_parse_ontology_has_iri_label + cls_labels_1 = get_class_by_lang('http://aims.fao.org/aos/agrovoc/c_3376', 'APTO', + requested_lang: :'pt-br').label + + assert_includes cls_labels_1 , 'Hortaliça de folha' + + cls_labels_2 = get_class_by_lang('http://aims.fao.org/aos/agrovoc/c_3376', 'APTO', + requested_lang: :en).label + + assert_includes cls_labels_2, 'Leaf vegetable' + end + def test_requested_language_found - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: :FR) assert_equal 'industrialisation', cls.prefLabel assert_equal ['développement industriel'], cls.synonym @@ -38,7 +55,7 @@ def test_requested_language_found assert_equal ['développement industriel'], properties.select { |x| x.to_s['altLabel'] }.values.first.map(&:to_s) assert_equal ['industrialisation'], properties.select { |x| x.to_s['prefLabel'] }.values.first.map(&:to_s) - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: :EN) assert_equal 'industrialization', cls.prefLabel assert_equal ['industrial development'], cls.synonym @@ -47,18 +64,19 @@ def test_requested_language_found assert_equal ['industrial development'], properties.select { |x| x.to_s['altLabel'] }.values.first.map(&:to_s) assert_equal ['industrialization'], properties.select { |x| x.to_s['prefLabel'] }.values.first.map(&:to_s) - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_13078', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_13078', 'INRAETHES', requested_lang: :FR) assert_equal 'carbone renouvelable', cls.prefLabel end def test_requeststore_not_set - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + skip 'Needs to be fixed in the future for Virtuoso' + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: nil) assert_equal 'industrialization', cls.prefLabel assert_equal cls.prefLabel, cls.prefLabel(include_languages: true) - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: :ALL) assert_equal 'industrialization', cls.prefLabel assert_equal Hash, cls.prefLabel(include_languages: true).class @@ -69,7 +87,7 @@ def test_requeststore_not_set def test_requested_language_not_found - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: :ES) assert_nil cls.prefLabel assert_empty cls.synonym @@ -79,48 +97,18 @@ def test_requested_language_not_found assert_empty properties.select { |x| x.to_s['prefLabel'] }.values end - def test_context_language - cls = get_class('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES') - cls.submission.bring_remaining - cls.submission.ontology.bring_remaining - - # Default portal language - cls.bring_remaining - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls, all: 'all')) - assert_equal response["@context"]["@language"], Goo.main_languages.first.to_s - assert_equal "http://www.w3.org/2000/01/rdf-schema#parents", response["@context"]["parents"] + def test_request_multiple_languages - # Request specific language - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls, lang: 'fr')) - assert_equal response["@context"]["@language"], 'fr' - - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls, lang: %w[fr en es])) - assert_equal response["@context"]["@language"], %w[fr en es] - - # Submission Natural Language - s = cls.submission - s.naturalLanguage = %w[fr en] - s.save - - cls = get_class('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES') - cls.submission.bring_remaining - cls.submission.ontology.bring_remaining - - # Default get submission first Natural Language - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls)) - assert_equal response["@context"]["@language"], 'fr' - - # Request specific language - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls, lang: 'fr')) - assert_equal response["@context"]["@language"], 'fr' - - response = MultiJson.load(LinkedData::Serializers::JSON.serialize(cls, lang: %w[fr en es])) - assert_equal response["@context"]["@language"], %w[fr en es] + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', + requested_lang: [:EN, :FR]) + pref_label_all_languages = { en: 'industrialization', fr: 'industrialisation' } + assert_includes pref_label_all_languages.values, cls.prefLabel + assert_equal pref_label_all_languages, cls.prefLabel(include_languages: true) end def test_request_all_languages - cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', + cls = get_class_by_lang('http://opendata.inrae.fr/thesaurusINRAE/c_22817', 'INRAETHES', requested_lang: :ALL) pref_label_all_languages = { en: 'industrialization', fr: 'industrialisation' } @@ -161,9 +149,9 @@ def get_class(cls, ont) LinkedData::Models::Class.find(cls).in(sub).first end - def get_class_by_lang(cls, requested_lang:, portal_languages: nil) + def get_class_by_lang(cls, ont, requested_lang:, portal_languages: nil) lang_set requested_lang: requested_lang, portal_languages: portal_languages - cls = get_class(cls, 'INRAETHES') + cls = get_class(cls, ont) refute_nil cls cls.bring_remaining cls.bring :unmapped diff --git a/test/models/test_ontology.rb b/test/models/test_ontology.rb index 68a038948..b3cef4b74 100644 --- a/test/models/test_ontology.rb +++ b/test/models/test_ontology.rb @@ -5,7 +5,7 @@ class TestOntology < LinkedData::TestOntologyCommon def self.before_suite - url , @@thread, @@port= self.new('').start_server + _, @@thread, @@port = self.new('').start_server end def self.after_suite @@ -159,6 +159,20 @@ def test_ontology_properties assert_equal "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#AlgorithmPurpose", props[0].id.to_s assert_equal "http://www.w3.org/2004/02/skos/core#altLabel", props[1].id.to_s + props[2].bring(*[:domain,:range]) + props[2].bring(:unmapped) + + assert_equal "http://bioontology.org/ontologies/biositemap.owl#biositemap_author", props[2].id.to_s + assert_equal "http://bioontology.org/ontologies/biositemap.owl#Resource_Description", props[2].domain + assert_equal "http://www.w3.org/2001/XMLSchema#string", props[2].range + refute_empty props[2].properties + + p = ont.property(props[2].id.to_s, display_all_attributes: true) + assert_equal "http://bioontology.org/ontologies/biositemap.owl#biositemap_author", p.id.to_s + assert_equal "http://bioontology.org/ontologies/biositemap.owl#Resource_Description", p.domain + assert_equal "http://www.w3.org/2001/XMLSchema#string", p.range + refute_empty p.properties + datatype_props = [] object_props = [] annotation_props = [] @@ -353,13 +367,13 @@ def test_ontology_lifecycle }) # Create - assert_equal false, o.exist? + assert_equal false, o.exist?(reload=true) o.save - assert_equal true, o.exist? + assert_equal true, o.exist?(reload=true) # Delete o.delete - assert_equal false, o.exist? + assert_equal false, o.exist?(reload=true) end def test_next_submission_id @@ -427,4 +441,66 @@ def test_duplicate_contacts assert sub.contact.length == 1 end + # A test to benchmark the time taken by bring_remaining (query not optimized, can take a long time if a lot of value in the list attributes) + def test_ontology_bring_remaining + # Creating the users + user1 = LinkedData::Models::User.new(:username => "user1", :email => "some1@email.org" ) + user1.passwordHash = "some random pass hash" + user1.save + user2 = LinkedData::Models::User.new(:username => "user2", :email => "some2@email.org" ) + user2.passwordHash = "some random pass hash" + user2.save + user3 = LinkedData::Models::User.new(:username => "user3", :email => "some3@email.org" ) + user3.passwordHash = "some random pass hash" + user3.save + user4 = LinkedData::Models::User.new(:username => "user4", :email => "some4@email.org" ) + user4.passwordHash = "some random pass hash" + user4.save + user5 = LinkedData::Models::User.new(:username => "user5", :email => "some5@email.org" ) + user5.passwordHash = "some random pass hash" + user5.save + user6 = LinkedData::Models::User.new(:username => "user6", :email => "some6@email.org" ) + user6.passwordHash = "some random pass hash" + user6.save + user7 = LinkedData::Models::User.new(:username => "user7", :email => "some7@email.org" ) + user7.passwordHash = "some random pass hash" + user7.save + + # Creating the categories + category1 = LinkedData::Models::Category.new({:name => "Test Category", :description => "This is a test category", :acronym => "TCG"}) + category1.save + category2 = LinkedData::Models::Category.new({:name => "Test2 Category", :description => "This is a test category", :acronym => "TCG2"}) + category2.save + category3 = LinkedData::Models::Category.new({:name => "Test2 Category", :description => "This is a test category", :acronym => "TCG3"}) + category3.save + category4 = LinkedData::Models::Category.new({:name => "Test2 Category", :description => "This is a test category", :acronym => "TCG4"}) + category4.save + category5 = LinkedData::Models::Category.new({:name => "Test2 Category", :description => "This is a test category", :acronym => "TCG5"}) + category5.save + category6 = LinkedData::Models::Category.new({:name => "Test2 Category", :description => "This is a test category", :acronym => "TCG6"}) + category6.save + + # Creating the groups + group1 = LinkedData::Models::Group.new({:name => "Test1 Group", :description => "This is a test group", :acronym => "TESTG1"}) + group1.save + group2 = LinkedData::Models::Group.new({:name => "Test2 Group", :description => "This is a test group", :acronym => "TESTG2"}) + group2.save + o = LinkedData::Models::Ontology.new({ + acronym: "TEST1", + administeredBy: [user1, user2, user3, user4, user5, user6, user7], + name: "test 1", + hasDomain: [category1, category2, category3, category4, category5, category6], + group: [group1, group2] + }) + o.save + + otest = LinkedData::Models::Ontology.find("TEST1").first + + Benchmark.bm do |x| + x.report { otest.bring_remaining } + end + + assert_equal "test 1", otest.name + end + end diff --git a/test/models/test_ontology_common.rb b/test/models/test_ontology_common.rb index f85b9dc3a..431a4ba4c 100644 --- a/test/models/test_ontology_common.rb +++ b/test/models/test_ontology_common.rb @@ -3,6 +3,7 @@ module LinkedData class TestOntologyCommon < LinkedData::TestCase + def create_count_mapping count = LinkedData::Models::MappingCount.where.all.length unless count > 2 @@ -11,6 +12,7 @@ def create_count_mapping end count end + def submission_dependent_objects(format, acronym, user_name, name_ont) #ontology format owl = LinkedData::Models::OntologyFormat.where(:acronym => format).first @@ -52,6 +54,10 @@ def submission_dependent_objects(format, acronym, user_name, name_ont) # delete = true # delete any existing submissions ############################################## def submission_parse(acronym, name, ontologyFile, id, parse_options={}, set_attributes={}) + if Goo.backend_vo? + old_slices = Goo.slice_loading_size + Goo.slice_loading_size = 20 + end return if ENV["SKIP_PARSING"] parse_options[:process_rdf].nil? && parse_options[:process_rdf] = true parse_options[:index_search].nil? && parse_options[:index_search] = false @@ -116,6 +122,10 @@ def submission_parse(acronym, name, ontologyFile, id, parse_options={}, set_attr rescue Exception => e puts "Error, logged in #{tmp_log.instance_variable_get("@logdev").dev.path}" raise e + ensure + if Goo.backend_vo? + Goo.slice_loading_size = old_slices + end end end @@ -247,6 +257,7 @@ def port_in_use?(port) true end end + end end diff --git a/test/models/test_ontology_submission.rb b/test/models/test_ontology_submission.rb index 1a6a60377..2399694b6 100644 --- a/test/models/test_ontology_submission.rb +++ b/test/models/test_ontology_submission.rb @@ -133,7 +133,7 @@ def test_skos_ontology roots.each do |root| q_broader = <<-eos SELECT ?children WHERE { - ?children #{RDF::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } + ?children #{RDF::Vocab::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } eos children_query = [] Goo.sparql_query_client.query(q_broader).each_solution do |sol| @@ -411,7 +411,7 @@ def test_process_submission_archive assert old_sub.zipped? assert File.file?(old_sub.uploadFilePath) - LinkedData::Models::OntologySubmission.const_set(:FILE_SIZE_ZIPPING_THRESHOLD, old_threshold) + LinkedData::Services::OntologySubmissionArchiver.const_set(:FILE_SIZE_ZIPPING_THRESHOLD, old_threshold) end def test_submission_diff_across_ontologies @@ -441,8 +441,8 @@ def test_index_properties submission_parse("BRO", "BRO Ontology", "./test/data/ontology_files/BRO_v3.5.owl", 1, process_rdf: true, extract_metadata: false, index_properties: true) - res = LinkedData::Models::Class.search("*:*", {:fq => "submissionAcronym:\"BRO\"", :start => 0, :rows => 80}, :property) - assert_equal 84, res["response"]["numFound"] + res = LinkedData::Models::OntologyProperty.search("*:*", {:fq => "submissionAcronym:\"BRO\"", :start => 0, :rows => 80}) + assert_equal 84, res["response"]["numFound"] # if 81 if owlapi import skos properties found = 0 res["response"]["docs"].each do |doc| @@ -466,12 +466,11 @@ def test_index_properties break if found == 2 end - assert_equal 2, found # if owliap does not import skos properties + assert_includes [1,2], found # if owliap does not import skos properties ont = LinkedData::Models::Ontology.find('BRO').first ont.unindex_properties(true) - - res = LinkedData::Models::Class.search("*:*", {:fq => "submissionAcronym:\"BRO\""},:property) + res = LinkedData::Models::OntologyProperty.search("*:*", {:fq => "submissionAcronym:\"BRO\""}) assert_equal 0, res["response"]["numFound"] end @@ -486,7 +485,7 @@ def test_index_multilingual doc = res["response"]["docs"].select{|doc| doc["resource_id"].to_s.eql?('http://bioontology.org/ontologies/Activity.owl#Activity')}.first refute_nil doc - assert_equal 30, doc.keys.select{|k| k['prefLabel'] || k['synonym']}.size # test that all the languages are indexed + assert_equal 12, doc.keys.select{|k| k['prefLabel'] || k['synonym']}.size # test that all the languages are indexed res = LinkedData::Models::Class.search("prefLabel_none:Activity", {:fq => "submissionAcronym:BRO", :start => 0, :rows => 80}) refute_equal 0, res["response"]["numFound"] @@ -502,10 +501,6 @@ def test_index_multilingual res = LinkedData::Models::Class.search("prefLabel_fr:Activity", {:fq => "submissionAcronym:BRO", :start => 0, :rows => 80}) assert_equal 0, res["response"]["numFound"] - - res = LinkedData::Models::Class.search("prefLabel_ja:カタログ", {:fq => "submissionAcronym:BRO", :start => 0, :rows => 80}) - refute_equal 0, res["response"]["numFound"] - refute_nil res["response"]["docs"].select{|doc| doc["resource_id"].eql?('http://bioontology.org/ontologies/Activity.owl#Catalog')}.first end def test_submission_parse_multilingual @@ -668,10 +663,10 @@ def test_submission_parse_gzip def test_download_ontology_file begin - server_url, server_thread, server_port = start_server + server_url, server_thread, _ = start_server sleep 3 # Allow the server to startup assert(server_thread.alive?, msg="Rack::Server thread should be alive, it's not!") - ont_count, ont_names, ont_models = create_ontologies_and_submissions(ont_count: 1, submission_count: 1) + _, _, ont_models = create_ontologies_and_submissions(ont_count: 1, submission_count: 1) ont = ont_models.first assert(ont.instance_of?(LinkedData::Models::Ontology), "ont is not an ontology: #{ont}") sub = ont.bring(:submissions).submissions.first @@ -810,10 +805,15 @@ def test_custom_property_generation #either the RDF label of the synonym assert ("rdfs label value" == c.prefLabel || "syn for class 6" == c.prefLabel) end + if c.id.to_s.include? "class3" assert_equal "class3", c.prefLabel end + if c.id.to_s.include? "class1" + + # binding.pry + assert_equal "class 1 literal", c.prefLabel end end @@ -1180,6 +1180,8 @@ def test_submission_metrics # See https://github.com/ncbo/ncbo_cron/issues/82#issuecomment-3104054081 def test_disappearing_values + skip "This issue no longer occurs with the latest goo/sparql-client from AgroPortal" + acronym = "ONTOMATEST" name = "ONTOMA Test Ontology" ontologyFile = "./test/data/ontology_files/OntoMA.1.1_vVersion_1.1_Date__11-2011.OWL" diff --git a/test/models/test_provisional_class.rb b/test/models/test_provisional_class.rb index 12062b99f..72eea506f 100644 --- a/test/models/test_provisional_class.rb +++ b/test/models/test_provisional_class.rb @@ -3,7 +3,8 @@ class TestProvisionalClass < LinkedData::TestOntologyCommon def self.before_suite - @@user = LinkedData::Models::User.new({username: "Test User", email: "tester@example.org", password: "password"}) + @@user = LinkedData::Models::User.new({username: "test_user_prov_class", + email: "test_user_prov_class@example.org", password: "password"}) @@user.save ont_count, ont_names, ont_models = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, @@ -24,12 +25,10 @@ def self.after_suite LinkedData::Models::ProvisionalClass.indexClear LinkedData::Models::ProvisionalClass.indexCommit LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - user = LinkedData::Models::User.find("Test User").first + user = LinkedData::Models::User.find("test_user_prov_class").first user.delete unless user.nil? - end - def test_provisional_class_lifecycle label = "Test Provisional Class Lifecycle" pc = LinkedData::Models::ProvisionalClass.new({label: label, :creator => @@user}) @@ -285,11 +284,11 @@ def test_provisional_class_search_indexing pc = @@provisional_class pc.ontology = @@ontology pc.unindex - resp = LinkedData::Models::Ontology.search("\"#{pc.label}\"", params) + resp = LinkedData::Models::ProvisionalClass.search("\"#{pc.label}\"", params) assert_equal 0, resp["response"]["numFound"] pc.index - resp = LinkedData::Models::Ontology.search("\"#{pc.label}\"", params) + resp = LinkedData::Models::ProvisionalClass.search("\"#{pc.label}\"", params) assert_equal 1, resp["response"]["numFound"] assert_equal pc.label, resp["response"]["docs"][0]["prefLabel"].first pc.unindex @@ -312,18 +311,18 @@ def test_provisional_class_search_indexing pc3.save pc3 = LinkedData::Models::ProvisionalClass.find(pc3.id).include(:label).first - resp = LinkedData::Models::Ontology.search("\"#{pc1.label}\"", params) + resp = LinkedData::Models::ProvisionalClass.search("\"#{pc1.label}\"", params) assert_equal 1, resp["response"]["numFound"] assert_equal pc1.label, resp["response"]["docs"][0]["prefLabel"].first par_len = resp["response"]["docs"][0]["parents"].length assert_equal 5, par_len assert_equal 1, (resp["response"]["docs"][0]["parents"].select { |x| x == class_id.to_s }).length - resp = LinkedData::Models::Ontology.search("\"#{pc2.label}\"", params) + resp = LinkedData::Models::ProvisionalClass.search("\"#{pc2.label}\"", params) assert_equal par_len + 1, resp["response"]["docs"][0]["parents"].length assert_equal 1, (resp["response"]["docs"][0]["parents"].select { |x| x == pc1.id.to_s }).length - resp = LinkedData::Models::Ontology.search("\"#{pc3.label}\"", params) + resp = LinkedData::Models::ProvisionalClass.search("\"#{pc3.label}\"", params) assert_equal par_len + 2, resp["response"]["docs"][0]["parents"].length assert_equal 1, (resp["response"]["docs"][0]["parents"].select { |x| x == pc1.id.to_s }).length assert_equal 1, (resp["response"]["docs"][0]["parents"].select { |x| x == pc2.id.to_s }).length diff --git a/test/models/test_provisional_relation.rb b/test/models/test_provisional_relation.rb index 78839f754..6c7d594f6 100644 --- a/test/models/test_provisional_relation.rb +++ b/test/models/test_provisional_relation.rb @@ -3,13 +3,16 @@ class TestProvisionalRelation < LinkedData::TestCase def self.before_suite - @@user = LinkedData::Models::User.new({username: "Test User", email: "tester@example.org", password: "password"}) + @@user = LinkedData::Models::User.new({username: "test_user_prov_relation", + email: "test_user_prov_relation@example.org", password: "password"}) @@user.save - ont_count, ont_names, ont_models = self.new('').create_ontologies_and_submissions(ont_count: 1, submission_count: 1, - process_submission: true, - process_options: {process_rdf: true, - extract_metadata: false}) + _, _, ont_models = self.new('').create_ontologies_and_submissions( + ont_count: 1, + submission_count: 1, + process_submission: true, + process_options: {process_rdf: true, extract_metadata: false} + ) @@ontology = ont_models.first @@ontology.bring(:name) @@ -30,6 +33,27 @@ def self.before_suite @@provisional_rel2.save end + def self.after_suite + if defined?(@@provisional_rel1) + rel = LinkedData::Models::ProvisionalRelation.find(@@provisional_rel1.id).first + rel.delete unless rel.nil? + end + + if defined?(@@provisional_rel2) + rel = LinkedData::Models::ProvisionalRelation.find(@@provisional_rel2.id).first + rel.delete unless rel.nil? + end + + if defined?(@@provisional_class) + pc = LinkedData::Models::ProvisionalClass.find(@@provisional_class.id).first + pc.delete unless pc.nil? + end + + LinkedData::SampleData::Ontology.delete_ontologies_and_submissions + + user = LinkedData::Models::User.find("test_user_prov_relation").first + user.delete unless user.nil? + end def test_create_provisional_relation rel1 = LinkedData::Models::ProvisionalRelation.find(@@provisional_rel1.id).first @@ -68,6 +92,4 @@ def test_target_class assert_equal @@ontology.acronym, target_class.submission.ontology.acronym end - - -end \ No newline at end of file +end diff --git a/test/models/test_resource.rb b/test/models/test_resource.rb new file mode 100644 index 000000000..63d51f660 --- /dev/null +++ b/test/models/test_resource.rb @@ -0,0 +1,320 @@ +require_relative "../test_case" +require_relative './test_ontology_common' + +class TestResource < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + + # Example + data = %( + . + "John Doe" . + "30"^^ . + "male" . + . + . + _:blanknode1 . + _:blanknode2 . + "test/bonjour"@fr . + "Person#human"@en . + "2024-11-29"^^ . + "http://person1.org/test" . + + _:blanknode1 "Jane Smith" . + _:blanknode1 "25"^^ . + _:blanknode1 "female" . + _:blanknode1 . + _:blanknode2 "Jane Smith 2" . + "Hiking" . + "Cooking" . + + . + "Alice Cooper" . + "35"^^ . + "female" . + . + _:skill1, _:skill2 . + _:skill1 "Programming" . + _:skill1 _:skill2 . + _:skill2 "Data Analysis" . + _:skill2 . + "Hiking" . + "Cooking" . + "Photography" . + + . + . + . + + ) + + graph = "http://example.org/test_graph" + Goo.sparql_data_client.execute_append_request(graph, data, '') + + # instance the resource model + @@resource1 = LinkedData::Models::Resource.new("http://example.org/test_graph", "http://example.org/person1") + end + + def self.after_suite + Goo.sparql_data_client.delete_graph("http://example.org/test_graph") + Goo.sparql_data_client.delete_graph("http://data.bioontology.org/ontologies/TEST-TRIPLES/submissions/2") + @resource1&.destroy + end + + def test_generate_model + @object = @@resource1.to_object + @model = @object.class + + assert_equal LinkedData::Models::Base, @model.ancestors[1] + + @model.model_settings[:attributes].map do |property, val| + property_url = "#{val[:property]}#{property}" + assert_includes @@resource1.to_hash.keys, property_url + + hash_value = @@resource1.to_hash[property_url] + object_value = @object.send(property.to_sym) + if property.to_sym == :knows + assert_equal hash_value.map{|x| x.is_a?(Hash) ? x.values : x}.flatten.map(&:to_s).sort, + object_value.map{|x| x.is_a?(String) ? x : x.to_h.values}.flatten.map(&:to_s).sort + elsif property.to_sym == :birthday + assert_equal Array(hash_value).map(&:object), Array(object_value) + else + assert_equal Array(hash_value).map(&:to_s), Array(object_value).map(&:to_s) + end + end + + assert_equal "http://example.org/person1", @object.id.to_s + + assert_equal Goo.namespaces[:foaf][:Person].to_s, @model.type_uri.to_s + end + + def test_resource_fetch_related_triples + result = @@resource1.to_hash + assert_instance_of Hash, result + + refute_empty result + + expected_result = { + "id" => "http://example.org/person1", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" => "http://xmlns.com/foaf/0.1/Person", + "http://xmlns.com/foaf/0.1/gender" => "male", + "http://xmlns.com/foaf/0.1/hasInterest" => %w[Cooking Hiking], + "http://xmlns.com/foaf/0.1/age" => "30", + "http://xmlns.com/foaf/0.1/birthday"=>"2024-11-29", + "http://xmlns.com/foaf/0.1/email" => "mailto:john@example.com", + "http://www.w3.org/2004/02/skos/core#altLabel" => "test/bonjour", + "http://www.w3.org/2004/02/skos/core#prefLabel" => "Person#human", + "http://xmlns.com/foaf/0.1/knows" => + ["http://example.org/person3", + { + "http://xmlns.com/foaf/0.1/gender" => "female", + "http://xmlns.com/foaf/0.1/age" => "25", + "http://xmlns.com/foaf/0.1/email" => "mailto:jane@example.com", + "http://xmlns.com/foaf/0.1/name" => "Jane Smith" + }, + { + "http://xmlns.com/foaf/0.1/name" => "Jane Smith 2" + } + ], + "http://xmlns.com/foaf/0.1/name" => "John Doe", + "http://xmlns.com/foaf/0.1/homepage"=>"http://person1.org/test", + "reverse" => { + "http://example2.org/person2" => "http://xmlns.com/foaf/0.1/mother", + "http://example2.org/person5" => ["http://xmlns.com/foaf/0.1/brother", "http://xmlns.com/foaf/0.1/friend"] + } + } + result = JSON.parse(MultiJson.dump(result)) + a = sort_nested_hash(result) + b = sort_nested_hash(expected_result) + assert_equal b, a + end + + def test_resource_serialization_json + result = @@resource1.to_json + + refute_empty result + expected_result = %( + { + "@context": {"ns0": "http://example.org/", "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "foaf": "http://xmlns.com/foaf/0.1/", "ns1": "http://example2.org/", "skos":"http://www.w3.org/2004/02/skos/core#"}, + "@graph": + [ + { + "@id": "ns0:person1", + "@type": "foaf:Person", + "foaf:name": "John Doe", + "foaf:birthday":{"@value":"2024-11-29", "@type":"http://www.w3.org/2001/XMLSchema#date"}, + "foaf:hasInterest":["Cooking", "Hiking"], + "foaf:age": {"@type": "http://www.w3.org/2001/XMLSchema#integer", "@value": "30"}, + "foaf:email": {"@id": "mailto:john@example.com"}, + "foaf:gender": "male", + "skos:altLabel":{"@value":"test/bonjour", "@language":"fr"}, + "skos:prefLabel":{"@value":"Person#human", "@language":"en"}, + "foaf:knows": [{"@id": "ns0:person3"}, {"@id": "_:g447140"}, {"@id": "_:g447160"}], + "foaf:homepage":"http://person1.org/test"}, + { + "@id": "_:g447140", + "foaf:name": "Jane Smith", + "foaf:age": {"@type": "http://www.w3.org/2001/XMLSchema#integer", "@value": "25"}, + "foaf:email": {"@id": "mailto:jane@example.com"}, + "foaf:gender": "female" + }, + {"@id": "_:g447160", "foaf:name": "Jane Smith 2"}, + {"@id": "ns1:person5", "foaf:friend": {"@id": "ns0:person1"}, "foaf:brother": {"@id": "ns0:person1"}}, + {"@id": "ns1:person2", "foaf:mother": {"@id": "ns0:person1"}} + ] + } + ) + result = JSON.parse(result.gsub(' ', '').gsub("\n", '').gsub(/_:g\d+/, 'blanke_nodes')) + expected_result = JSON.parse(expected_result.gsub(' ', '').gsub("\n", '').gsub(/_:g\d+/, 'blanke_nodes')) + + a = sort_nested_hash(result) + b = sort_nested_hash(expected_result) + + assert_equal b, a + end + + def test_resource_serialization_xml + result = @@resource1.to_xml + + refute_empty result + expected_result = %( + + + 2024-11-29 + male + Cooking + Hiking + 30 + + test/bonjour + Person#human + + + + female + 25 + + Jane Smith + + + + + Jane Smith 2 + + + John Doe + http://person1.org/test + + + + + + + + + + ) + a = result.gsub(' ', '').gsub(/rdf:nodeID="[^"]*"/, '').split("\n").reject(&:empty?) + b = expected_result.gsub(' ', '').gsub(/rdf:nodeID="[^"]*"/, '').split("\n").reject(&:empty?) + + assert_equal b.sort, a.sort + end + + def test_resource_serialization_ntriples + result = @@resource1.to_ntriples + + refute_empty result + + expected_result = %( + . + "2024-11-29"^^ . + "male" . + "Cooking" . + "Hiking" . + "30"^^ . + . + "test/bonjour"@fr . + "Person#human"@en . + . + _:g447260 "female" . + _:g447260 "25"^^ . + _:g447260 . + _:g447260 "Jane Smith" . + _:g447260 . + _:g447280 "Jane Smith 2" . + _:g447280 . + "John Doe" . + "http://person1.org/test" . + . + . + . + ) + a = result.gsub(' ', '').gsub(/_:g\d+/, 'blanke_nodes').split("\n").reject(&:empty?) + b = expected_result.gsub(' ', '').gsub(/_:g\d+/, 'blanke_nodes').split("\n").reject(&:empty?) + + assert_equal b.sort, a.sort + end + + def test_resource_serialization_turtle + result = @@resource1.to_turtle + refute_empty result + expected_result = %( + @prefix rdf: . + @prefix ns0: . + @prefix foaf: . + @prefix skos: . + @prefix ns1: . + + ns0:person1 + a foaf:Person ; + skos:altLabel "test/bonjour"@fr ; + skos:prefLabel "Person#human"@en ; + foaf:age 30 ; + foaf:birthday "2024-11-29"^^ ; + foaf:email ; + foaf:gender "male" ; + foaf:hasInterest "Cooking", "Hiking" ; + foaf:homepage "http://person1.org/test" ; + foaf:knows ns0:person3, [ + foaf:age 25 ; + foaf:email ; + foaf:gender "female" ; + foaf:name "Jane Smith" + ], [ + foaf:name "Jane Smith 2" + ] ; + foaf:name "John Doe" . + + ns1:person2 + foaf:mother ns0:person1 . + + ns1:person5 + foaf:brother ns0:person1 ; + foaf:friend ns0:person1 . + ) + a = result.gsub(' ', '').split("\n").reject(&:empty?) + b = expected_result.gsub(' ', '').split("\n").reject(&:empty?) + + assert_equal b.sort, a.sort + end + + private + + def sort_nested_hash(hash) + sorted_hash = {} + + hash.each do |key, value| + if value.is_a?(Hash) + sorted_hash[key] = sort_nested_hash(value) + elsif value.is_a?(Array) + sorted_hash[key] = value.map { |item| item.is_a?(Hash) ? sort_nested_hash(item) : item }.sort_by { |item| item.to_s } + else + sorted_hash[key] = value + end + end + + sorted_hash.sort.to_h + end + +end \ No newline at end of file diff --git a/test/models/test_search.rb b/test/models/test_search.rb new file mode 100644 index 000000000..c25c74113 --- /dev/null +++ b/test/models/test_search.rb @@ -0,0 +1,172 @@ +require_relative '../test_case' + +class TestSearch < LinkedData::TestCase + + def self.after_suite + backend_4s_delete + LinkedData::Models::Ontology.indexClear + Goo.search_client(:ontology_data)&.clear_all_data + end + + def setup + self.class.after_suite + end + + def test_search_ontology + _, _, created_ontologies = create_ontologies_and_submissions({ + process_submission: true, + process_options: { + process_rdf: true, + generate_missing_labels: false, + extract_metadata: false, run_metrics: true + }, + acronym: 'BROTEST', + name: 'ontTEST Bla', + file_path: '../../../../test/data/ontology_files/BRO_v3.2.owl', + ont_count: 2, + submission_count: 2 + }) + ontologies = LinkedData::Models::Ontology.search('*:*', { fq: 'resource_model: "ontology"' })['response']['docs'] + + assert_equal 2, ontologies.size + ontologies.each do |ont| + select_ont = created_ontologies.select { |ont_created| ont_created.id.to_s.eql?(ont['id']) }.first + refute_nil select_ont + select_ont.bring_remaining + assert_equal ont['name_text'], select_ont.name + assert_equal ont['acronym_text'], select_ont.acronym + assert_equal ont['viewingRestriction_t'], select_ont.viewingRestriction + assert_equal ont['ontologyType_t'], select_ont.ontologyType.id + end + + submissions = LinkedData::Models::Ontology.search('*:*', { fq: 'resource_model: "ontology_submission"' })['response']['docs'] + assert_equal 4, submissions.size + submissions.each do |sub| + created_sub = LinkedData::Models::OntologySubmission.find(RDF::URI.new(sub['id'])).first&.bring_remaining + refute_nil created_sub + assert_equal sub['description_text'], created_sub.description + assert_equal sub['submissionId_i'], created_sub.submissionId + assert_equal sub['uri_text'], created_sub.uri + assert_equal sub['status_t'], created_sub.status + assert_equal sub['deprecated_b'], created_sub.deprecated + assert_equal sub['hasOntologyLanguage_t'], created_sub.hasOntologyLanguage.id.to_s + assert_equal sub['released_dt'], created_sub.released.utc.strftime('%Y-%m-%dT%H:%M:%SZ') + assert_equal sub['creationDate_dt'], created_sub.creationDate.utc.strftime('%Y-%m-%dT%H:%M:%SZ') + assert_equal(sub['contact_txt'], created_sub.contact.map { |x| x.bring_remaining.embedded_doc }) + assert_equal sub['dataDump_t'], created_sub.dataDump + assert_equal sub['csvDump_t'], created_sub.csvDump + assert_equal sub['uriLookupEndpoint_t'], created_sub.uriLookupEndpoint + assert_equal sub['openSearchDescription_t'], created_sub.openSearchDescription + assert_equal sub['uploadFilePath_t'], created_sub.uploadFilePath + assert_equal sub['submissionStatus_txt'].sort, created_sub.submissionStatus.map { |x| x.id.to_s }.sort + + created_sub.metrics.bring_remaining + + assert_equal sub['metrics_classes_i'], created_sub.metrics.classes + assert_equal sub['metrics_individuals_i'], created_sub.metrics.individuals + assert_equal sub['metrics_properties_i'], created_sub.metrics.properties + assert_equal sub['metrics_maxDepth_i'], created_sub.metrics.maxDepth + assert_equal sub['metrics_maxChildCount_i'], created_sub.metrics.maxChildCount + assert_equal sub['metrics_averageChildCount_i'], created_sub.metrics.averageChildCount + assert_equal sub['metrics_classesWithOneChild_i'], created_sub.metrics.classesWithOneChild + assert_equal sub['metrics_classesWithMoreThan25Children_i'], created_sub.metrics.classesWithMoreThan25Children + assert_equal sub['metrics_classesWithNoDefinition_i'], created_sub.metrics.classesWithNoDefinition + + embed_doc = created_sub.ontology.bring_remaining.embedded_doc + embed_doc.each do |k, v| + if v.is_a?(Array) + assert_equal v, Array(sub["ontology_#{k}"]) + else + assert_equal v, sub["ontology_#{k}"] + end + end + end + end + + def test_search_ontology_data + create_ontologies_and_submissions({ + process_submission: true, + process_options: { + process_rdf: true, + extract_metadata: false, + generate_missing_labels: false, + index_all_data: true + }, + acronym: 'BROTEST', + name: 'ontTEST Bla', + file_path: 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + ont_count: 1, + submission_count: 1, + ontology_format: 'SKOS' + }) + ont_sub = LinkedData::Models::Ontology.find('BROTEST-0').first + ont_sub = ont_sub.latest_submission + + refute_empty(ont_sub.submissionStatus.select { |x| x.id['INDEXED_ALL_DATA'] }) + + conn = Goo.search_client(:ontology_data) + submission_fq = "submission_id_t:\"#{ont_sub.id}\"" + + count_ids = Goo.sparql_query_client.query("SELECT (COUNT( DISTINCT ?id) as ?c) FROM <#{ont_sub.id}> WHERE {?id ?p ?v}") + .first[:c] + .to_i + + total_triples = Goo.sparql_query_client.query("SELECT (COUNT(*) as ?c) FROM <#{ont_sub.id}> WHERE {?s ?p ?o}").first[:c].to_i + + response = conn.search('*', fq: submission_fq, rows: count_ids + 100) + # Count only RDF predicate-derived fields: + # - `type_t` / `type_txt` + # - escaped URI predicate fields (e.g., `http___..._t` / `http___..._txt`) + # This avoids counting Solr metadata/copy fields whose presence varies by schema. + rdf_field = lambda do |field_name| + field_name == 'type_t' || + field_name == 'type_txt' || + (field_name.end_with?('_t', '_txt') && field_name.match?(/\A[a-z]+___/)) + end + index_total_triples = response['response']['docs'].sum do |doc| + doc.sum do |k, v| + rdf_field.call(k) ? Array(v).size : 0 + end + end + + # TODO: fix maybe in future sometime randomly don't index excactly all the triples + assert_in_delta total_triples, index_total_triples, 200 + assert_in_delta count_ids, response['response']['numFound'], 100 + + response = conn.search('*', fq: [submission_fq, 'resource_id:"http://opendata.inrae.fr/thesaurusINRAE/c_10065"']) + + assert_equal 1, response['response']['numFound'] + doc = response['response']['docs'].first + + expected_doc = { + 'id' => 'http://opendata.inrae.fr/thesaurusINRAE/c_10065_BROTEST-0', + 'submission_id_t' => 'http://data.bioontology.org/ontologies/BROTEST-0/submissions/1', + 'ontology_t' => 'BROTEST-0', + 'resource_id' => 'http://opendata.inrae.fr/thesaurusINRAE/c_10065', + 'type_txt' => %w[http://www.w3.org/2004/02/skos/core#Concept http://www.w3.org/2002/07/owl#NamedIndividual], + 'http___www.w3.org_2004_02_skos_core_inScheme_txt' => %w[http://opendata.inrae.fr/thesaurusINRAE/thesaurusINRAE http://opendata.inrae.fr/thesaurusINRAE/mt_53], + 'http___www.w3.org_2004_02_skos_core_broader_t' => 'http://opendata.inrae.fr/thesaurusINRAE/c_9937', + 'http___www.w3.org_2004_02_skos_core_altLabel_txt' => ['GMO food', + 'aliment transgénique', + 'aliment OGM', + 'transgenic food'], + 'http___www.w3.org_2004_02_skos_core_prefLabel_txt' => ['genetically modified food', + 'aliment génétiquement modifié'], + 'resource_model' => 'ontology_submission' + } + + doc.delete('_version_') + + assert_equal expected_doc['id'], doc['id'] + assert_equal expected_doc['submission_id_t'], doc['submission_id_t'] + assert_equal expected_doc['ontology_t'], doc['ontology_t'] + assert_equal expected_doc['resource_id'], doc['resource_id'] + assert_equal expected_doc['type_txt'].sort, doc['type_txt'].sort + assert_equal expected_doc['http___www.w3.org_2004_02_skos_core_inScheme_txt'].sort, doc['http___www.w3.org_2004_02_skos_core_inScheme_txt'].sort + assert_equal expected_doc['http___www.w3.org_2004_02_skos_core_broader_t'], doc['http___www.w3.org_2004_02_skos_core_broader_t'] + assert_equal expected_doc['http___www.w3.org_2004_02_skos_core_altLabel_txt'].sort, doc['http___www.w3.org_2004_02_skos_core_altLabel_txt'].sort + assert_equal expected_doc['http___www.w3.org_2004_02_skos_core_prefLabel_txt'].sort, doc['http___www.w3.org_2004_02_skos_core_prefLabel_txt'].sort + assert_equal expected_doc['resource_model'], doc['resource_model'] + end + +end diff --git a/test/models/test_skos_submission.rb b/test/models/test_skos_submission.rb index 35bfeee29..ef11f17f2 100644 --- a/test/models/test_skos_submission.rb +++ b/test/models/test_skos_submission.rb @@ -43,7 +43,7 @@ def test_roots_no_main_scheme roots.each do |root| q_broader = <<-eos SELECT ?children WHERE { - ?children #{RDF::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } + ?children #{RDF::Vocab::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } eos children_query = [] Goo.sparql_query_client.query(q_broader).each_solution do |sol| @@ -92,7 +92,7 @@ def test_roots_of_multiple_scheme roots.each do |r| selected_schemes = r.inScheme.select { |s| concept_schemes.include?(s) } refute_empty selected_schemes - assert_equal r.isInActiveScheme, selected_schemes + assert_equal r.isInActiveScheme.sort, selected_schemes.sort assert_equal r.isInActiveCollection, [] end roots = roots.map { |r| r.id.to_s } unless roots.nil? diff --git a/test/models/test_slice.rb b/test/models/test_slice.rb index 8cc243b10..d8b173f75 100644 --- a/test/models/test_slice.rb +++ b/test/models/test_slice.rb @@ -70,6 +70,19 @@ def test_synchronization assert slices.map {|s| s.acronym}.include?(@@group_acronym) end + def test_synchronization_skips_groups_without_ontologies + empty_group = LinkedData::Models::Group.new({ + acronym: "NOONTS", + name: "Group Without Ontologies" + }).save + + LinkedData::Models::Slice.synchronize_groups_to_slices + + assert_nil LinkedData::Models::Slice.find("noonts").first + ensure + empty_group&.delete + end + def test_slice_acronym_validity s = LinkedData::Models::Slice.new({ :name => "Test Slice", @@ -100,4 +113,4 @@ def self._create_group }).save end -end \ No newline at end of file +end diff --git a/test/models/user/test_subscription.rb b/test/models/user/test_subscription.rb index 93c710958..b28219199 100644 --- a/test/models/user/test_subscription.rb +++ b/test/models/user/test_subscription.rb @@ -3,9 +3,27 @@ class TestSubscription < LinkedData::TestCase def self.before_suite - @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first - @@ont.bring_remaining - @@user = @@ont.administeredBy.first + @@ont = LinkedData::SampleData::Ontology + .create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first + @@ont.bring(:administeredBy) + + candidate = @@ont.administeredBy&.first + @@user = candidate && LinkedData::Models::User.find(candidate.id) + .include(:username, :email, :passwordHash, :subscription) + .first + + unless @@user&.valid? + suffix = SecureRandom.hex(4) + @@user = LinkedData::Models::User.new( + username: "subscription_user_#{suffix}", + email: "subscription_user_#{suffix}@example.org", + password: "password" + ) + @@user.save + @@ont.administeredBy = [@@user] + @@ont.save + end + @@user.bring_remaining end @@ -30,8 +48,8 @@ def _subscription(ont, type = "ALL") end def _delete_subscriptions - if defined?(@@subscriptions) - @@subscriptions.each {|s| s.delete} + if self.class.class_variable_defined?(:@@subscriptions) + @@subscriptions.each { |s| s.delete } end @@subscriptions = nil end diff --git a/test/models/user/test_user_oauth.rb b/test/models/user/test_user_oauth.rb new file mode 100644 index 000000000..0d8378ad1 --- /dev/null +++ b/test/models/user/test_user_oauth.rb @@ -0,0 +1,80 @@ +require_relative '../../test_case' + +class TestUserOAuthAuthentication < LinkedData::TestCase + + def self.before_suite + @@fake_responses = { + github: { + id: 123456789, + login: 'github_user', + email: 'github_user@example.com', + name: 'GitHub User', + avatar_url: 'https://avatars.githubusercontent.com/u/123456789' + }, + google: { + sub: 'google_user_id', + email: 'google_user@example.com', + name: 'Google User', + given_name: 'Google', + family_name: 'User', + picture: 'https://lh3.googleusercontent.com/a-/user-profile-image-url' + }, + orcid: { + orcid: '0000-0002-1825-0097', + email: 'orcid_user@example.com', + name: { + "family-name": 'ORCID', + "given-names": 'User' + } + } + } + end + + + def test_authentication_new_users + users = [] + + @@fake_responses.each do |provider, data| + WebMock.stub_request(:get, LinkedData::Models::User.oauth_providers[provider][:link]) + .to_return(status: 200, body: data.to_json, headers: { 'Content-Type' => 'application/json' }) + user = LinkedData::Models::User.oauth_authenticate('fake token', provider) + refute_nil user + assert user.is_a?(LinkedData::Models::User) + assert_equal user.email, data[:email] + users << user + end + + users.each(&:delete) + end + + def test_authentication_existent_users + users = [] + @@fake_responses.each do |provider, data| + user_hash = LinkedData::Models::User.send("user_from_#{provider}_data", data.stringify_keys) + + user = LinkedData::Models::User.new(user_hash) + user.githubId = nil + user.orcidId = nil + user.password = 'password' + + assert user.valid? + + user.save + + WebMock.stub_request(:get, LinkedData::Models::User.oauth_providers[provider][:link]) + .to_return(status: 200, body: data.to_json, headers: { 'Content-Type' => 'application/json' }) + auth_user = LinkedData::Models::User.oauth_authenticate('fake token', provider) + + assert_equal auth_user.id, user.id + + if provider.eql?(:github) + assert_equal data[:login], auth_user.githubId + elsif provider.eql?(:orcid) + assert_equal data[:orcid], auth_user.orcidId + end + users << user + end + users.each(&:delete) + end + +end diff --git a/test/rack/test_request_authorization.rb b/test/rack/test_request_authorization.rb index 76fc7af9a..f087c50b2 100644 --- a/test/rack/test_request_authorization.rb +++ b/test/rack/test_request_authorization.rb @@ -1,15 +1,15 @@ -require 'minitest/unit' +require 'minitest/autorun' require "rack/test" require "json" require "logger" require_relative "../../lib/ontologies_linked_data" -require_relative "../../config/config.rb" +require_relative "../../config/config.test" LOGGER = Logger.new($stdout) ENV["rack.test"] = "true" -class TestRackAuthorization < MiniTest::Unit::TestCase +class TestRackAuthorization < Minitest::Test include Rack::Test::Methods def app @@ -63,10 +63,12 @@ def test_authorize assert_equal 401, last_response.status get "/ontologies", {}, {"Authorization" => "bogus auth header"} assert_equal 401, last_response.status + get "/ontologies", {}, {"Authorization" => 'apikey token=bugutoken'} + assert_equal 401, last_response.status get "/ontologies", {}, {"Authorization" => 'apikey token="'+@apikey+''+'"'} assert_equal 200, last_response.status apikey = MultiJson.load(last_response.body) - assert @apikey.eql?(apikey) + assert_equal @apikey, apikey get "/ontologies", {}, {"Authorization" => "apikey token=#{@apikey}"} assert_equal 200, last_response.status apikey = MultiJson.load(last_response.body) @@ -75,6 +77,8 @@ def test_authorize assert_equal 200, last_response.status apikey = MultiJson.load(last_response.body) assert_equal @apikey, apikey + get "/ontologies", {}, {"Authorization" => "apikey token=#{@apikey}&userapikey=bogusapikey"} + assert_equal 401, last_response.status get "/ontologies", {}, {"Authorization" => 'apikey token="'+@apikey+'&userapikey='+@userapikey+'"'} assert_equal 200, last_response.status apikey = MultiJson.load(last_response.body) diff --git a/test/rack/test_request_formats.rb b/test/rack/test_request_formats.rb index 5af4a629c..6a955f1c3 100644 --- a/test/rack/test_request_formats.rb +++ b/test/rack/test_request_formats.rb @@ -1,9 +1,9 @@ -require 'minitest/unit' +require 'minitest/autorun' require "rack/test" require "multi_json" require_relative "../../lib/ontologies_linked_data" -class TestLinkedDataSerializer < MiniTest::Unit::TestCase +class TestLinkedDataSerializer < Minitest::Test include Rack::Test::Methods def app diff --git a/test/serializer/test_serializer_json.rb b/test/serializer/test_serializer_json.rb index 128e5dd1f..c0fede1ff 100644 --- a/test/serializer/test_serializer_json.rb +++ b/test/serializer/test_serializer_json.rb @@ -1,9 +1,9 @@ -require 'minitest/unit' +require 'minitest/autorun' require "multi_json" require_relative "../../lib/ontologies_linked_data" -require_relative "../../config/config" +require_relative "../../config/config.test" -class TestSerializerOutput < MiniTest::Unit::TestCase +class TestSerializerOutput < Minitest::Test class Car < LinkedData::Models::Base model :car, name_with: :model, namespace: :omv attribute :model, enforce: [:unique] diff --git a/test/serializer/test_serializer_xml.rb b/test/serializer/test_serializer_xml.rb index 9d5d8c027..a07d1611f 100644 --- a/test/serializer/test_serializer_xml.rb +++ b/test/serializer/test_serializer_xml.rb @@ -1,9 +1,9 @@ -require 'minitest/unit' +require 'minitest/autorun' require "date" require "multi_json" require_relative "../../lib/ontologies_linked_data" -class TestSerializerXML < MiniTest::Unit::TestCase +class TestSerializerXML < Minitest::Test class Person attr_accessor :name def initialize(name) diff --git a/test/serializer/test_to_flex_hash.rb b/test/serializer/test_to_flex_hash.rb index 7070e22b8..c0704c69c 100644 --- a/test/serializer/test_to_flex_hash.rb +++ b/test/serializer/test_to_flex_hash.rb @@ -1,8 +1,8 @@ -require 'minitest/unit' +require 'minitest/autorun' require 'pry' require_relative "../../lib/ontologies_linked_data" -class ToHashTest < MiniTest::Unit::TestCase +class ToHashTest < Minitest::Test class Person include LinkedData::Hypermedia::Resource diff --git a/test/test_case.rb b/test/test_case.rb index df7d5b766..a72400e57 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -18,29 +18,48 @@ require_relative 'test_log_file' require_relative '../lib/ontologies_linked_data' +require_relative '../config/config.test' +require 'minitest/autorun' +require 'webmock/minitest' +WebMock.allow_net_connect! -if ENV['OVERRIDE_CONFIG'] == 'true' - SOLR_HOST = ENV.include?('SOLR_HOST') ? ENV['SOLR_HOST'] : 'localhost' - - LinkedData.config do |config| - config.goo_backend_name = ENV['GOO_BACKEND_NAME'] - config.goo_port = ENV['GOO_PORT'].to_i - config.goo_host = ENV['GOO_HOST'] - config.goo_path_query = ENV['GOO_PATH_QUERY'] - config.goo_path_data = ENV['GOO_PATH_DATA'] - config.goo_path_update = ENV['GOO_PATH_UPDATE'] - config.goo_redis_host = ENV['REDIS_HOST'] - config.goo_redis_port = ENV['REDIS_PORT'] - config.http_redis_host = ENV['REDIS_HOST'] - config.http_redis_port = ENV['REDIS_PORT'] - config.search_server_url = "http://#{SOLR_HOST}:8983/solr/term_search_core1" - config.property_search_server_url = "http://#{SOLR_HOST}:8983/solr/prop_search_core1" +# Minitest 5+ no longer supports `MiniTest::Unit.runner=`. We emulate the old +# custom runner behavior using Minitest hooks and a small patch around suite runs. +module LinkedData + module MinitestSuiteHooks + def run_suite(reporter, options = {}) + return if filter_runnable_methods(options).empty? + + before_suite if respond_to?(:before_suite) + super + rescue Exception => e + puts e.message + puts e.backtrace.join("\n\t") + puts 'Traced from:' + raise + ensure + after_suite if respond_to?(:after_suite) + end end end -require_relative '../config/config' -require 'minitest/unit' -MiniTest::Unit.autorun +# Apply per-suite before/after hooks around every runnable (test class). +Minitest::Runnable.singleton_class.prepend(LinkedData::MinitestSuiteHooks) + +# Emulate the old Unit runner's before/after suites behavior. +# (Not all Minitest versions expose `before_run` / `after_run`.) +module LinkedData + module MinitestRunHooks + def run(*args) + LinkedData::TestCase.backend_4s_delete + super + ensure + LinkedData::TestCase.backend_4s_delete + end + end +end + +Minitest.singleton_class.prepend(LinkedData::MinitestRunHooks) # Check to make sure you want to run if not pointed at localhost safe_hosts = Regexp.new(/localhost|-ut|ncbo-dev*|ncbo-unittest*/) @@ -69,40 +88,8 @@ def safe_redis_hosts?(sh) end module LinkedData - class Unit < MiniTest::Unit - def before_suites - # code to run before the first test (gets inherited in sub-tests) - end - def after_suites - # code to run after the last test (gets inherited in sub-tests) - end - - def _run_suites(suites, type) - TestCase.backend_4s_delete - before_suites - super(suites, type) - ensure - TestCase.backend_4s_delete - after_suites - end - - def _run_suite(suite, type) - suite.before_suite if suite.respond_to?(:before_suite) - super(suite, type) - rescue Exception => e - puts e.message - puts e.backtrace.join("\n\t") - puts 'Traced from:' - raise e - ensure - suite.after_suite if suite.respond_to?(:after_suite) - end - end - - MiniTest::Unit.runner = LinkedData::Unit.new - - class TestCase < MiniTest::Unit::TestCase + class TestCase < Minitest::Test # Ensure all threads exit on any exception Thread.abort_on_exception = true @@ -238,7 +225,11 @@ def self.backend_4s_delete raise StandardError, 'Too many triples in KB, does not seem right to run tests' unless count_pattern('?s ?p ?o') < 400000 - Goo.sparql_update_client.update('DELETE {?s ?p ?o } WHERE { ?s ?p ?o }') + graphs = Goo.sparql_query_client.query("SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o . } }") + graphs.each_solution do |sol| + Goo.sparql_data_client.delete_graph(sol[:g]) + end + LinkedData::Models::SubmissionStatus.init_enum LinkedData::Models::OntologyType.init_enum LinkedData::Models::OntologyFormat.init_enum @@ -246,4 +237,5 @@ def self.backend_4s_delete LinkedData::Models::Users::NotificationType.init_enum end end + end diff --git a/test/util/test_file_helpers.rb b/test/util/test_file_helpers.rb index 0d0362625..1406dd1bd 100644 --- a/test/util/test_file_helpers.rb +++ b/test/util/test_file_helpers.rb @@ -1,7 +1,7 @@ require 'minitest/autorun' require_relative '../../lib/ontologies_linked_data/utils/file' -class TestUtilsFile < MiniTest::Unit::TestCase +class TestUtilsFile < Minitest::Test DST = nil def setup diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index e590fe9cc..a0f952367 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -3,6 +3,7 @@ require 'email_spec' require 'logger' require 'mocha/minitest' +require 'securerandom' class TestNotifications < LinkedData::TestCase include EmailSpec::Helpers @@ -125,8 +126,7 @@ def test_disable_administrative_notifications ont.latest_submission(status: :any).process_submission(Logger.new(TestLogFile.new)) admin_mails = LinkedData::Utils::Notifier.ontology_admin_emails(ont) assert_equal 1, all_emails.size, 'number of send emails' - - refute_match @@support_mails, last_email_sent.to.sort + assert_empty(last_email_sent.to & @@support_mails) assert_equal admin_mails, last_email_sent.to.sort assert_match 'Parsing Success', all_emails.last.subject LinkedData.settings.enable_administrative_notifications = true @@ -135,17 +135,16 @@ def test_disable_administrative_notifications end def test_remote_ontology_pull_notification - recipients = ['test@example.org'] _ont_count, _acronyms, ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions( - ont_count: 1, submission_count: 1, process_submission: false + ont_count: 1, submission_count: 1, acronym: "PULL-NOTIF-ONT", process_submission: false ) - ont = LinkedData::Models::Ontology.find(ontologies[0].id) .include(:acronym, :administeredBy, :name, :submissions).first + suffix = SecureRandom.hex(4) ont_admins = Array.new(3) { LinkedData::Models::User.new } ont_admins.each_with_index do |user, i| - user.username = "Test User #{i}" - user.email = "tester_#{i}@example.org" + user.username = "notif_test_user_#{suffix}_#{i}" + user.email = "notif_test_user_#{suffix}_#{i}@example.org" user.password = 'password' user.save assert user.valid?, user.errors @@ -166,7 +165,7 @@ def test_remote_ontology_pull_notification assert_equal admin_mails, last_email_sent.to.sort ensure ont_admins.each do |user| - user&.delete + user.delete if user&.persistent? end end