diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 39d28a8c5..75ad06930 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -28,6 +28,7 @@ jobs: bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Run unit tests run: | + cp .env.sample .env ci_env=`bash <(curl -s https://codecov.io/env)` GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} diff --git a/Capfile b/Capfile index 7ecc995cd..95799ba1b 100644 --- a/Capfile +++ b/Capfile @@ -22,6 +22,5 @@ require 'capistrano/bundler' # require 'capistrano/rails/assets' # require 'capistrano/rails/migrations' require 'capistrano/locally' -require 'new_relic/recipes' # announce deployments in NewRelic # Loads custom tasks from `lib/capistrano/tasks' if you have any defined. Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r } diff --git a/Gemfile b/Gemfile index be9a23d56..183344ca2 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ end gem 'request_store' gem 'parallel' -gem 'google-protobuf' +gem 'google-protobuf', '~> 4.30.0' gem 'net-ftp' gem 'json-ld', '~> 3.2.0' gem 'rdf-raptor', github:'ruby-rdf/rdf-raptor', ref: '6392ceabf71c3233b0f7f0172f662bd4a22cd534' # use version 3.3.0 when available @@ -38,7 +38,7 @@ gem 'redis' gem 'redis-store' # Monitoring -gem 'newrelic_rpm', group: [:default, :deployment] +gem "sentry-ruby", "~> 5.24" # HTTP server gem 'unicorn' @@ -53,7 +53,7 @@ gem 'pandoc-ruby' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ontoportal-lirmm/ncbo_ontology_recommender.git', branch: 'development' -gem 'ontologies_linked_data', github: 'earthportal/ontologies_linked_data', branch: 'feature/projects' +gem 'ontologies_linked_data', github: 'earthportal/ontologies_linked_data', branch: 'development' gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' @@ -90,4 +90,4 @@ group :test do gem 'simplecov-cobertura' # for codecov.io gem 'webmock' gem 'webrick' -end +end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index cdaf15169..7c586f4ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/earthportal/ontologies_linked_data.git - revision: a3692de0d091e5ed6f9a048e409d2c5760e6947f - branch: feature/projects + revision: 96fe46904909f9a2f4b9360d9a0cf2ced6a0ecbe + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -20,7 +20,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: e48a2d13a65cc2dd1c12d116cfc9da9061106861 + revision: 04680ed78dfd98cfe004d9a1d7019f3f06e9b667 branch: development specs: goo (0.0.2) @@ -38,7 +38,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git - revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + revision: aeb0222400f1b423cb865545c41233d2cbd82bfc branch: development specs: ncbo_annotator (0.0.1) @@ -49,7 +49,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: cc4cd9218db7181c4843772631b7f3a96c74a4aa + revision: df22084bd5960254cc21408f1090a7faf9e3ab72 branch: master specs: ncbo_cron (0.0.1) @@ -103,7 +103,7 @@ GIT GIT remote: https://github.com/sinatra/sinatra.git - revision: cfcc70dee1133690207b5a3dc6000426ec04e250 + revision: 5e1598501eb23a8673d61034df7be7d50c228400 specs: rack-protection (4.1.1) base64 (>= 0.1.0) @@ -146,14 +146,14 @@ GEM sshkit (>= 1.6.1, != 1.7.0) ansi (1.5.0) ast (2.4.3) - base64 (0.2.0) + base64 (0.3.0) bcp47_spec (0.2.1) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) - benchmark (0.4.0) - bigdecimal (3.1.9) + benchmark (0.4.1) + bigdecimal (3.2.2) builder (3.3.0) capistrano (3.19.2) airbrussh (>= 1.0.0) @@ -169,7 +169,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.3.5) - connection_pool (2.5.0) + connection_pool (2.5.3) crack (0.4.5) rexml dante (0.2.0) @@ -177,23 +177,23 @@ GEM declarative (0.0.20) docile (1.4.1) domain_name (0.6.20240107) - drb (2.2.1) - ed25519 (1.3.0) + drb (2.2.3) + ed25519 (1.4.0) et-orbi (1.2.11) tzinfo - faraday (2.12.2) + faraday (2.13.1) faraday-net_http (>= 2.0, < 3.5) json logger - faraday-net_http (3.4.0) + faraday-net_http (3.4.1) net-http (>= 0.5.0) - faraday-retry (2.2.1) + faraday-retry (2.3.2) faraday (~> 2.0) ffi (1.15.5) fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) - gapic-common (0.25.0) + gapic-common (1.0.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) google-cloud-env (~> 2.2) @@ -205,15 +205,15 @@ GEM grpc (~> 1.66) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.7.0) + google-analytics-data (0.7.1) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.16.0) - gapic-common (>= 0.25.0, < 2.a) + google-analytics-data-v1beta (0.17.0) + gapic-common (~> 1.0) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.16.0) + google-apis-analytics_v3 (0.17.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.16.0) + google-apis-core (0.18.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -224,34 +224,34 @@ GEM google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.2.2) + google-cloud-env (2.3.1) base64 (~> 0.2) faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) - google-logging-utils (0.1.0) - google-protobuf (4.30.1) + google-logging-utils (0.2.0) + google-protobuf (4.30.2) bigdecimal rake (>= 13) - google-protobuf (4.30.1-aarch64-linux) + google-protobuf (4.30.2-aarch64-linux) bigdecimal rake (>= 13) - google-protobuf (4.30.1-arm64-darwin) + google-protobuf (4.30.2-arm64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.30.1-x86-linux) + google-protobuf (4.30.2-x86-linux) bigdecimal rake (>= 13) - google-protobuf (4.30.1-x86_64-darwin) + google-protobuf (4.30.2-x86_64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.30.1-x86_64-linux) + google-protobuf (4.30.2-x86_64-linux) bigdecimal rake (>= 13) - googleapis-common-protos (1.7.0) + googleapis-common-protos (1.8.0) google-protobuf (>= 3.18, < 5.a) - googleapis-common-protos-types (~> 1.7) + googleapis-common-protos-types (~> 1.20) grpc (~> 1.41) - googleapis-common-protos-types (1.19.0) + googleapis-common-protos-types (1.20.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.14.0) faraday (>= 1.0, < 3.a) @@ -261,28 +261,28 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.71.0) + grpc (1.73.0) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-aarch64-linux) + grpc (1.73.0-aarch64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-arm64-darwin) + grpc (1.73.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-x86-linux) + grpc (1.73.0-x86-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-x86_64-darwin) + grpc (1.73.0-x86_64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-x86_64-linux) + grpc (1.73.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt - hashdiff (1.1.2) + hashdiff (1.2.0) htmlentities (4.3.4) http-accept (1.7.0) http-cookie (1.0.8) @@ -291,7 +291,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.10.2) + json (2.12.2) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -303,14 +303,14 @@ GEM json-schema (5.1.1) addressable (~> 2.8) bigdecimal (~> 3.1) - jwt (2.10.1) + jwt (2.10.2) base64 kgio (2.11.4) - language_server-protocol (3.17.0.4) - libxml-ruby (5.0.3) + language_server-protocol (3.17.0.5) + libxml-ruby (5.0.4) link_header (0.0.8) lint_roller (1.1.0) - logger (1.6.6) + logger (1.7.0) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -319,10 +319,10 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.6.1) + mime-types (3.7.0) logger - mime-types-data (~> 3.2015) - mime-types-data (3.2025.0318) + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2025.0701) mini_mime (1.1.5) minitest (5.25.5) minitest-fail-fast (0.1.0) @@ -346,9 +346,9 @@ GEM time net-http (0.6.0) uri - net-http-persistent (4.0.5) - connection_pool (~> 2.2) - net-imap (0.5.6) + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-imap (0.5.9) date net-protocol net-pop (0.1.2) @@ -363,29 +363,29 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.17.0) - oj (3.16.10) + oj (3.16.11) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) logger os (1.1.4) - ostruct (0.6.1) + ostruct (0.6.2) pandoc-ruby (2.1.10) - parallel (1.26.3) + parallel (1.27.0) parseconfig (1.1.2) - parser (3.3.7.2) + parser (3.3.8.0) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) + prism (1.4.0) pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.12) + rack (3.1.16) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -394,11 +394,12 @@ GEM rack (>= 0.4) rack-contrib (2.5.0) rack (< 4) - rack-cors (2.0.2) - rack (>= 2.0.0) - rack-mini-profiler (3.3.1) + rack-cors (3.0.0) + logger + rack (>= 3.0.14) + rack-mini-profiler (4.0.0) rack (>= 1.2.0) - rack-session (2.1.0) + rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) @@ -408,11 +409,13 @@ GEM rack (>= 3) rainbow (3.1.1) raindrops (0.20.1) - rake (13.2.1) - rdf (3.3.2) + rake (13.3.0) + rdf (3.3.3) bcp47_spec (~> 0.2) bigdecimal (~> 3.1, >= 3.1.5) link_header (~> 0.0, >= 0.0.8) + logger (~> 1.5) + ostruct (~> 0.6) rdf-rdfxml (3.3.0) builder (~> 3.2, >= 3.2.4) htmlentities (~> 4.3) @@ -425,7 +428,7 @@ GEM rexml (~> 3.2) redis (5.4.0) redis-client (>= 0.22.0) - redis-client (0.24.0) + redis-client (0.25.0) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -449,7 +452,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.74.0) + rubocop (1.77.0) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -457,11 +460,12 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.45.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.41.0) + rubocop-ast (1.45.1) parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -469,7 +473,10 @@ GEM rufus-scheduler (3.9.2) fugit (~> 1.1, >= 1.11.1) securerandom (0.4.1) - signet (0.19.0) + sentry-ruby (5.26.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -545,7 +552,7 @@ DEPENDENCIES ed25519 (>= 1.2, < 2.0) ffi (~> 1.15.0) goo! - google-protobuf + google-protobuf (~> 4.30.0) haml (~> 5.2.2) json-ld (~> 3.2.0) json-schema @@ -559,7 +566,6 @@ DEPENDENCIES ncbo_cron! ncbo_ontology_recommender! net-ftp - newrelic_rpm oj ontologies_linked_data! pandoc-ruby @@ -584,6 +590,7 @@ DEPENDENCIES request_store rexml rubocop + sentry-ruby (~> 5.24) shotgun! simplecov simplecov-cobertura diff --git a/app.rb b/app.rb index be90bd342..1c8b5c3e9 100644 --- a/app.rb +++ b/app.rb @@ -135,8 +135,19 @@ require_relative 'config/unicorn_workerkiller' end -# Add New Relic last to allow Rack middleware instrumentation -require 'newrelic_rpm' +if $SENTRY_DSN + require 'sentry-ruby' + Sentry.init do |config| + config.dsn = $SENTRY_DSN + + # Add data like request headers and IP for users, + # see https://docs.sentry.io/platforms/ruby/data-management/data-collected/ for more info + config.send_default_pii = true + end + + # use Rack::RewindableInput::Middleware + use Sentry::Rack::CaptureExceptions +end # Initialize the app require_relative 'init' diff --git a/bin/ontoportal b/bin/ontoportal index 9bf8cf5ba..5d86cb147 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -78,7 +78,7 @@ build_docker_run_cmd() { local goo_path="$3" local sparql_client_path="$4" - local docker_run_cmd="docker compose -p ontoportal_docker run --rm -it --name api-service" + local docker_run_cmd="docker compose --profile vo -p ontoportal_docker run --rm -it --name api-service" local bash_cmd="" # Conditionally add bind mounts only if the paths are not empty @@ -108,7 +108,7 @@ provision() { source .env echo "[+] Cleaning volumes" - docker compose -f docker-compose.yml --profile 4store down --volumes >/dev/null 2>&1 + docker compose -f docker-compose.yml --profile vo down --volumes >/dev/null 2>&1 docker compose -p ontoportal_docker down --volumes >/dev/null 2>&1 commands=( @@ -119,7 +119,7 @@ provision() { ) for cmd in "${commands[@]}"; do echo "[+] Run: $cmd" - docker_cron_cmd="docker compose -f docker-compose.yml -p ontoportal_docker run --remove-orphans --rm --name cron-service --service-ports ncbo_cron bash -c \"$cmd\" >/dev/null 2>&1" + docker_cron_cmd="docker compose -f docker-compose.yml -p ontoportal_docker run --remove-orphans --rm --name cron-service --service-ports ncbo_cron bash -c \"$cmd\"" if ! eval "$docker_cron_cmd"; then echo "Error: Failed to run provisioning . $cmd" exit 1 diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 0eabcee81..0c6c874ec 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -22,6 +22,7 @@ REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PR SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr" SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr" +$SENTRY_DSN = ENV.include?("SENTRY_DSN") ? ENV["SENTRY_DSN"] : nil begin # For prefLabel extract main_lang first, or anything if no main found. # For other properties only properties with a lang that is included in main_lang are used diff --git a/controllers/application_controller.rb b/controllers/application_controller.rb index 5693f8c16..1b16a56d7 100644 --- a/controllers/application_controller.rb +++ b/controllers/application_controller.rb @@ -1,5 +1,8 @@ # This is the base class for controllers in the application. # Code in the before or after blocks will run on every request +require_relative '../helpers/swagger_ui_helper' +require_relative '../helpers/openapi_helper' + class ApplicationController include Sinatra::Delegator extend Sinatra::Delegator @@ -12,4 +15,87 @@ class ApplicationController after { } + register Sinatra::OpenAPIHelper + + configure do + set :app_name, 'MOD-API Documentation' + set :api_version, '1.0.0' + set :api_description, 'Ontoportal MOD-API documentation' + set :base_url, LinkedData.settings.rest_url_prefix + + set :api_schemas, { + hydraPage: { + type: 'object', + required: ['@context', '@id', '@type', 'totalItems', 'itemsPerPage', 'member', 'view'], + properties: { + '@context': { + type: 'object' + }, + '@id': { type: 'string', format: 'uri' }, + '@type': { type: 'string', enum: ['hydra:Collection'] }, + 'totalItems': { type: 'integer' }, + 'itemsPerPage': { type: 'integer' }, + 'view': { + type: 'object', + required: ['@id', '@type'], + properties: { + '@id': { type: 'string', format: 'uri' }, + '@type': { type: 'string', enum: ['hydra:PartialCollectionView'] }, + 'firstPage': { type: 'string', format: 'uri' }, + 'previousPage': { type: 'string', format: 'uri' }, + 'nextPage': { type: 'string', format: 'uri' }, + 'lastPage': { type: 'string', format: 'uri' } + } + }, + 'member': { + type: 'array', + items: { type: 'object' } + } + } + }, + modSemanticArtefact: { + type: 'object', + properties: { + '@id': { type: 'string', format: 'uri'}, + '@type': { type: 'string', const: 'https://w3id.org/mod#modSemanticArtefact' }, + links: { + type: 'object', + properties: { + link: { type: 'string', format: 'uri' }, + '@context': { type: 'array', items: {type: 'string'} } + } + }, + '@context': { + type: 'object', + properties: { + property: { type: 'string', format: 'uri' }, + } + } + } + }, + + modSemanticArtefactDistribution: { + type: 'object', + properties: { + '@id': { type: 'string', format: 'uri'}, + '@type': { type: 'string', const: 'https://w3id.org/mod#SemanticArtefactDistribution' }, + links: { + type: 'object', + properties: { + link: { type: 'string', format: 'uri' }, + '@context': { type: 'array', items: {type: 'string'} } + } + }, + '@context': { + type: 'object', + properties: { + property: { type: 'string', format: 'uri' }, + } + } + } + } + } + + end + end diff --git a/controllers/artefacts.rb b/controllers/artefacts.rb deleted file mode 100644 index 348f43340..000000000 --- a/controllers/artefacts.rb +++ /dev/null @@ -1,62 +0,0 @@ -class ArtefactsController < ApplicationController - - namespace "/artefacts" do - # Get all Semantic Artefacts - get do - check_last_modified_collection(LinkedData::Models::SemanticArtefact) - attributes, page, pagesize, _, _ = settings_params(LinkedData::Models::SemanticArtefact) - pagesize = 20 if params["pagesize"].nil? - artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) - reply artefacts - end - - # Get one semantic artefact by ID - get "/:artefactID" do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? - check_last_modified(artefact) - artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) - reply artefact - end - - # Display latest distribution - get "/:artefactID/distributions/latest" do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "You must provide a valid artefactID to retrieve an artefact" if artefact.nil? - include_status = params["include_status"] && !params["include_status"].empty? ? params["include_status"].to_sym : :any - latest_distribution = artefact.latest_distribution(status: include_status) - - if latest_distribution - check_last_modified(latest_distribution) - latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - end - reply latest_distribution - end - - # Display a distribution - get '/:artefactID/distributions/:distributionID' do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 422, "Semantic Artefact #{params["artefactID"]} does not exist" unless artefact - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - artefact_distribution = artefact.distribution(params["distributionID"]) - error 404, "Distribuution with #{params['distributionID']} not found" if artefact_distribution.nil? - artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - reply artefact_distribution - end - - # Display a distribution - get '/:artefactID/distributions' do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "Semantic Artefact #{params["acronym"]} does not exist" unless artefact - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - options = { - status: (params["include_status"] || "ANY"), - includes: LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) - } - distros = artefact.all_distributions(options) - reply distros.sort {|a,b| b.distributionId.to_i <=> a.distributionId.to_i } - end - - end - -end \ No newline at end of file diff --git a/controllers/classes_controller.rb b/controllers/classes_controller.rb index c774f6033..93361aa92 100644 --- a/controllers/classes_controller.rb +++ b/controllers/classes_controller.rb @@ -6,24 +6,20 @@ class ClassesController < ApplicationController get do includes_param_check ont, submission = get_ontology_and_submission - cls_count = submission.class_count(LOGGER) - error 403, "Unable to display classes due to missing metrics for #{submission.id.to_s}. Please contact the administrator." if cls_count < 0 - attributes, page, size, order_by_hash, bring_unmapped_needed = settings_params(LinkedData::Models::Class) + attributes, page, size, order_by_hash = settings_params(LinkedData::Models::Class).first(4) check_last_modified_segment(LinkedData::Models::Class, [ont.acronym]) index = LinkedData::Models::Class.in(submission) if order_by_hash index = index.order_by(order_by_hash) - cls_count = nil # Add index here when, indexing fixed # index_name = 'classes_sort_by_date' # index = index.index_as(index_name) # index = index.with_index(index_name) end - - page_data = index - page_data = page_data.include(attributes).page(page, size).page_count_set(cls_count).all + + page_data = index.include(attributes).page(page, size).all reply page_data end diff --git a/controllers/documentation_controller.rb b/controllers/documentation_controller.rb new file mode 100644 index 000000000..90d239b49 --- /dev/null +++ b/controllers/documentation_controller.rb @@ -0,0 +1,33 @@ +class DocumentationController < ApplicationController + get '/mod-api/doc/api' do + content_type 'text/html' + <<-HTML + + + + + MOD-API Documentation + + + +
+ + + + + HTML + end + + # Serve OpenAPI JSON + get '/openapi.json' do + content_type :json + generate_openapi_json.to_json + end +end diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 7f3280795..6df9b4c78 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -2,67 +2,38 @@ class HomeController < ApplicationController - CLASS_MAP = { - Property: 'LinkedData::Models::ObjectProperty' - } - namespace '/' do + doc('Catalog', 'Get the semantic artefact catalogue') do + default_params(display: true) + default_responses(success: true) + end get do - expires 3600, :public - last_modified @@root_last_modified ||= Time.now.httpdate - routes = routes_list - - # TODO: delete when ccv will be on production - routes.delete('/ccv') - - routes.delete('/resource_index') if LinkedData.settings.enable_resource_index == false - - routes.delete('/Agents') - - routes_hash = {} - context = {} - - routes.each do |route| - next unless routes_by_class.key?(route) - - route_no_slash = route.gsub('/', '') - context[route_no_slash] = routes_by_class[route].type_uri.to_s if routes_by_class[route].respond_to?(:type_uri) - routes_hash[route_no_slash] = LinkedData.settings.rest_url_prefix + route_no_slash - end - catalog_class = LinkedData::Models::SemanticArtefactCatalog catalog = catalog_class.all.first || create_catalog + check_last_modified(catalog) + attributes_to_include = includes_param[0] == :all ? catalog_class.attributes(:all) : catalog_class.goo_attrs_to_load(includes_param) catalog.bring(*attributes_to_include) - if catalog.loaded_attributes.include?(:federated_portals) - catalog.federated_portals = catalog.federated_portals.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - catalog.federated_portals.each { |item| item.delete('apikey') } - end - if catalog.loaded_attributes.include?(:fundedBy) - catalog.fundedBy = catalog.fundedBy.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - end - catalog.class.link_to *routes_hash.map { |key, url| LinkedData::Hypermedia::Link.new(key, url, context[key]) } - + catalog.federated_portals = safe_parse(catalog.federated_portals) { |item| item.delete('apikey') unless current_user&.admin? } if catalog.loaded_attributes.include?(:federated_portals) + catalog.fundedBy = safe_parse(catalog.fundedBy) if catalog.loaded_attributes.include?(:fundedBy) reply catalog end patch do + error 401, "Unauthorized: Admin access required to update the catalog" unless current_user&.admin? catalog = LinkedData::Models::SemanticArtefactCatalog.where.first error 422, "There is no catalog configs in the triple store" if catalog.nil? populate_from_params(catalog, params) if catalog.valid? catalog.save status 200 + reply catalog else error 422, catalog.errors end end - get "doc/api" do - redirect "/documentation", 301 - end - get "documentation" do @metadata_all = get_metadata_all.sort { |a, b| a[0].name <=> b[0].name } haml "documentation/documentation".to_sym, :layout => "documentation/layout".to_sym @@ -74,7 +45,7 @@ def create_catalog catalog = nil catalogs = LinkedData::Models::SemanticArtefactCatalog.all if catalogs.nil? || catalogs.empty? - catalog = instance_from_params(LinkedData::Models::SemanticArtefactCatalog, {"test_attr_to_persist" => "test_to_persist"}) + catalog = instance_from_params(LinkedData::Models::SemanticArtefactCatalog, {}) if catalog.valid? catalog.save else @@ -82,8 +53,29 @@ def create_catalog end end catalog - end - + end + + def safe_parse(value) + return nil unless value + + parse_item = ->(item) { + begin + parsed = JSON.parse( + item.gsub(/:(\w+)=>/, '"\1":').gsub('=>', ':').gsub('\"', '"') + ) + yield(parsed) if block_given? + parsed + rescue JSON::ParserError => e + nil + end + } + + if value.is_a?(Array) + value.map { |item| parse_item.call(item) } + else + parse_item.call(value) + end + end end end diff --git a/controllers/mod/artefacts_controller.rb b/controllers/mod/artefacts_controller.rb new file mode 100644 index 000000000..8eb16d03e --- /dev/null +++ b/controllers/mod/artefacts_controller.rb @@ -0,0 +1,92 @@ +class ArtefactsController < ApplicationController + namespace "/mod-api" do + namespace "/artefacts" do + + doc('Artefact', 'Get information about all semantic artefacts') do + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefact) + attributes, page, pagesize = settings_params(LinkedData::Models::SemanticArtefact).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefact.goo_attrs_to_load([]) if includes_param.first == :all + artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) + reply artefacts + end + + doc('Artefact', 'Get information about a semantic artefact') do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID" do + artefact = find_artefact(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? + check_last_modified(artefact) + artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) + reply artefact + end + + doc('Artefact', "Get information about a semantic artefact's latest distribution") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true) + end + get "/:artefactID/distributions/latest" do + artefact = find_artefact(params["artefactID"]) + include_status = params["include_status"]&.to_sym || :any + latest_distribution = artefact.latest_distribution(status: include_status) + + if latest_distribution + check_last_modified(latest_distribution) + latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + end + reply latest_distribution + end + + doc('Artefact', "Get information about a semantic artefact's distribution") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('distributionID', type: 'number', description: 'The id of the distribution', default: 5) + default_params(display: true) + default_responses(success: true, not_found: true) + end + get '/:artefactID/distributions/:distributionID' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + artefact_distribution = artefact.distribution(params["distributionID"]) + error 404, "Distribution with ID #{params['distributionID']} not found" if artefact_distribution.nil? + artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + reply artefact_distribution + end + + doc('Artefact', "Get information about a semantic artefact's distributions") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + default_responses(not_found: true) + end + get '/:artefactID/distributions' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactDistribution).first(3) + attributes = LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) if includes_param.first == :all + distros = artefact.all_distributions(attributes, page, pagesize) + reply distros + end + + doc('Record', "Get information about a semantic artefact catalog record") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID/record" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + end + end +end diff --git a/controllers/mod/mod_search_controller.rb b/controllers/mod/mod_search_controller.rb new file mode 100644 index 000000000..9b045ab7c --- /dev/null +++ b/controllers/mod/mod_search_controller.rb @@ -0,0 +1,33 @@ +class ModSearchController < ApplicationController + namespace "/mod-api" do + namespace "/search" do + + doc('Search', 'Search content/metadata of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + result = process_search + reply hydra_page_object(result.to_a, result.aggregate) + end + + doc('Search', 'Search content of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get '/content' do + result = process_search + reply hydra_page_object(result.to_a, result.aggregate) + end + + doc('Search', 'Search metadata of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get '/metadata' do + hydra_page_result = search_metadata + reply hydra_page_result + end + end + end +end \ No newline at end of file diff --git a/controllers/mod/records_controller.rb b/controllers/mod/records_controller.rb new file mode 100644 index 000000000..c8c3df208 --- /dev/null +++ b/controllers/mod/records_controller.rb @@ -0,0 +1,31 @@ +class RecordsController < ApplicationController + namespace "/mod-api" do + namespace "/records" do + doc('Record', "Get information about all semantic artefact catalog records") do + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefactCatalogRecord) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load([]) if includes_param.first == :all + records = LinkedData::Models::SemanticArtefactCatalogRecord.all(attributes, page, pagesize) + reply records + end + + doc('Record', "Get information about a semantic artefact catalog record") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + end + end +end diff --git a/controllers/mod/resources_controller.rb b/controllers/mod/resources_controller.rb new file mode 100644 index 000000000..72cc13002 --- /dev/null +++ b/controllers/mod/resources_controller.rb @@ -0,0 +1,90 @@ +class ResourcesController < ApplicationController + namespace "/mod-api" do + namespace "/artefacts/:artefactID/resources" do + + doc('Artefact', "Get a list of all the resources within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + _, page, size = settings_params(LinkedData::Models::Class).first(3) + size_per_type = [size / 6, 1].max + + types = [ + LinkedData::Models::Class, + LinkedData::Models::Instance, + LinkedData::Models::SKOS::Scheme, + LinkedData::Models::SKOS::Collection, + LinkedData::Models::SKOS::Label + ] + + total_count = 0 + resources = types.flat_map do |model| + resource_page = load_resources_hydra_page(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_type) + total_count += resource_page.aggregate + resource_page.to_a + end + + props_page = load_properties_hydra_page(ontology, latest_submission, page, size_per_type) + resources.concat(props_page.to_a) + total_count += props_page.aggregate + reply hydra_page_object(resources, total_count) + end + + def self.define_resource_routes(resource_types, expected_type) + resource_types.each do |type| + + doc('Artefact', "Get a list of all #{type} within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get "/#{type}" do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + model_class = (type == 'properties') ? LinkedData::Models::OntologyProperty : model_from_type(type) + attributes, page, size = settings_params(model_class).first(3) + + if type == 'properties' + reply load_properties_hydra_page(ontology, latest_submission, page, size) + else + rdf_type = LinkedData::Models::Class.class_rdf_type(latest_submission) + if rdf_type == expected_type + reply load_resources_hydra_page(ontology, latest_submission, model_class, attributes, page, size) + else + reply hydra_empty_page + end + end + end + + + doc('Artefact', "Get specific #{type} of a semantic artefact by it's uri") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('uri', type: 'string', description: 'The uri of the resource', default: "FAKE_URI") + default_responses(success: true, not_found: true) + end + get "/#{type}/:uri" do + reply resolve_resource_by_uri + end + end + end + + define_resource_routes(%w[classes individuals], RDF::OWL[:Class]) + define_resource_routes(%w[concepts schemes collections labels], RDF::Vocab::SKOS[:Concept]) + define_resource_routes(%w[properties], 'properties') + + + doc('Artefact', "Get a specific resources from within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('uri', type: 'string', description: 'The uri of the resource', default: "FAKE_URI") + default_responses(success: true, not_found: true) + end + get '/:uri' do + reply resolve_resource_by_uri + end + end + end +end diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 682bd7bf7..7661c94f8 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -4,65 +4,22 @@ class SearchController < ApplicationController namespace "/search" do # execute a search query + get do - process_search + page = process_search + reply 200, page end post do - process_search + page = process_search + reply 200, page end namespace "/ontologies" do get do query = params[:query] || params[:q] - groups = params.fetch("groups", "").split(',') - categories = params.fetch("hasDomain", "").split(',') - languages = params.fetch("languages", "").split(',') - status = params.fetch("status", "").split(',') - format = params.fetch("hasOntologyLanguage", "").split(',') - is_of_type = params.fetch("isOfType", "").split(',') - has_format = params.fetch("hasFormat", "").split(',') - visibility = params["visibility"] - show_views = params["show_views"] == 'true' - sort = params.fetch("sort", "score desc, ontology_name_sort asc, ontology_acronym_sort asc") - page, page_size = page_params - - fq = [ - 'resource_model:"ontology_submission"', - 'submissionStatus_txt:ERROR_* OR submissionStatus_txt:"RDF" OR submissionStatus_txt:"UPLOADED"', - groups.map { |x| "ontology_group_txt:\"http://data.bioontology.org/groups/#{x.upcase}\"" }.join(' OR '), - categories.map { |x| "ontology_hasDomain_txt:\"http://data.bioontology.org/categories/#{x.upcase}\"" }.join(' OR '), - languages.map { |x| "naturalLanguage_txt:\"#{x.downcase}\"" }.join(' OR '), - ] - - fq << "ontology_viewingRestriction_t:#{visibility}" unless visibility.blank? - fq << "!ontology_viewOf_t:*" unless show_views - - fq << format.map { |x| "hasOntologyLanguage_t:\"http://data.bioontology.org/ontology_formats/#{x}\"" }.join(' OR ') unless format.blank? - - fq << status.map { |x| "status_t:#{x}" }.join(' OR ') unless status.blank? - fq << is_of_type.map { |x| "isOfType_t:#{x}" }.join(' OR ') unless is_of_type.blank? - fq << has_format.map { |x| "hasFormalityLevel_t:#{x}" }.join(' OR ') unless has_format.blank? - - fq.reject!(&:blank?) - - if params[:qf] - qf = params[:qf] - else - qf = [ - "ontology_acronymSuggestEdge^25 ontology_nameSuggestEdge^15 descriptionSuggestEdge^10 ", # start of the word first - "ontology_acronym_text^15 ontology_name_text^10 description_text^5 ", # full word match - "ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last - ].join(' ') - end - - page_data = search(Ontology, query, { - fq: fq, - qf: qf, - page: page, - page_size: page_size, - sort: sort - }) + options = get_ontology_metadata_search_options(params) + page_data = search(Ontology, query, options) total_found = page_data.aggregate ontology_rank = LinkedData::Models::Ontology.rank @@ -171,90 +128,17 @@ class SearchController < ApplicationController sort = "score desc, acronym_sort asc, name_sort asc" end - reply 200, search(LinkedData::Models::Agent, + resp = search(LinkedData::Models::Agent, query, fq: fq, qf: qf, page: page, page_size: page_size, sort: sort) - end - end - - private - - def search(model, query, params = {}) - query = query.blank? ? "*" : query - - resp = model.search(query, search_params(**params)) - - total_found = resp["response"]["numFound"] - docs = resp["response"]["docs"] - - page_object(docs, total_found) - end - def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOperators: "true", page:, page_size:, fl: '*,score', sort:) - { - defType: defType, - fq: fq, - qf: qf, - sort: sort, - start: (page - 1) * page_size, - rows: page_size, - fl: fl, - stopwords: stopwords, - lowercaseOperators: lowercaseOperators, - } - end + agents = resp.map { |doc| build_agent_from_search_result(doc) } - def process_search(params = nil) - params ||= @params - text = params["q"] - - query = get_term_search_query(text, params) - # puts "Edismax query: #{query}, params: #{params}" - set_page_params(params) - - docs = Array.new - resp = LinkedData::Models::Class.search(query, params) - total_found = resp["response"]["numFound"] - add_matched_fields(resp, Sinatra::Helpers::SearchHelper::MATCH_TYPE_PREFLABEL) - ontology_rank = LinkedData::Models::Ontology.rank - - resp["response"]["docs"].each do |doc| - doc = doc.symbolize_keys - # NCBO-974 - doc[:matchType] = resp["match_types"][doc[:id]] - resource_id = doc[:resource_id] - doc.delete :resource_id - doc[:id] = resource_id - # TODO: The `rescue next` on the following line shouldn't be here - # However, at some point we didn't store the ontologyId in the index - # and these records haven't been cleared out so this is getting skipped - ontology_uri = doc[:ontologyId].sub(/\/submissions\/.*/, "") rescue next - ontology = LinkedData::Models::Ontology.read_only(id: ontology_uri, acronym: doc[:submissionAcronym]) - submission = LinkedData::Models::OntologySubmission.read_only(id: doc[:ontologyId], ontology: ontology) - doc[:submission] = submission - doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 - doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) - - doc = filter_attrs_by_language(doc) - - instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) - docs.push(instance) - end - unless params['sort'] - if !text.nil? && text[-1] == '*' - docs.sort! { |a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]] } - else - docs.sort! { |a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]] } - end + reply 200, page_object(agents, resp.aggregate) end - - # need to return a Page object - page = page_object(docs, total_found) - - reply 200, page end end diff --git a/docker-compose.yml b/docker-compose.yml index 07b0cda1e..1e8c1b607 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ x-app: &app - image: agroportal/ontologies_api:master + image: agroportal/ontologies_api:development environment: &env # default bundle config resolves to /usr/local/bundle/config inside of the container # we are setting it to local app directory if we need to use 'bundle config local' @@ -9,9 +9,12 @@ x-app: &app REDIS_PORT: 6379 SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr - GOO_BACKEND_NAME: 4store - GOO_PORT: 9000 - GOO_HOST: 4store-ut + GOO_BACKEND_NAME: virtuoso + GOO_PORT: 8890 + GOO_HOST: virtuoso-ut + GOO_PATH_DATA: /sparql/ + GOO_PATH_QUERY: /sparql/ + GOO_PATH_UPDATE: /sparql/ MGREP_HOST: mgrep-ut MGREP_PORT: 55555 REPOSITORY_FOLDER: /srv/ontoportal/data/repository @@ -32,8 +35,6 @@ services: <<: *env BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle - profiles: - - 4store depends_on: solr-ut: condition: service_healthy @@ -41,7 +42,7 @@ services: condition: service_healthy mgrep-ut: condition: service_started - 4store-ut: + virtuoso-ut: condition: service_started ncbo_cron: condition: service_started @@ -61,8 +62,6 @@ services: <<: *env BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_cron/.bundle command: "bundle exec bin/ncbo_cron" - profiles: - - 4store volumes: - app_cron:/srv/ontoportal/ncbo_cron - repository:/srv/ontoportal/data/repository @@ -77,7 +76,7 @@ services: condition: service_healthy mgrep-ut: condition: service_started - 4store-ut: + virtuoso-ut: condition: service_started @@ -159,8 +158,6 @@ services: ports: - 1111:1111 - 8890:8890 - profiles: - - vo healthcheck: test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] start_period: 10s diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index c65541410..856fcf71c 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -390,27 +390,28 @@ def retrieve_latest_submissions(options = {}) latest_submissions end - def get_ontology_and_submission - ont = Ontology.find(@params["ontology"]) + def get_ontology_and_submission(ontology_acronym: nil) + acronym = ontology_acronym || @params["ontology"] + ont = Ontology.find(acronym) .include(:acronym, :administeredBy, :acl, :viewingRestriction) .include(submissions: [:submissionId, submissionStatus: [:code], ontology: [:acronym], metrics: :classes]) .first - error(404, "Ontology '#{@params["ontology"]}' not found.") if ont.nil? + error(404, "Ontology (artefact) '#{acronym}' not found.") if ont.nil? check_access(ont) if LinkedData.settings.enable_security # Security check submission = nil if @params.include? "ontology_submission_id" submission = ont.submission(@params[:ontology_submission_id]) if submission.nil? error 404, - "You must provide an existing submission ID for the #{@params["acronym"]} ontology" + "You must provide an existing submission (distribution) ID for the #{acronym} ontology (artefact)" end else submission = ont.latest_submission(status: [:RDF]) end - error 404, "Ontology #{@params["ontology"]} submission not found." if submission.nil? + error 404, "Ontology (artefact) #{acronym} submission (distribution) not found." if submission.nil? if !submission.ready?(status: [:RDF]) - error 404, "Ontology #{@params["ontology"]} submission #{submission.submissionId} has not been parsed." + error 404, "Ontology (artefact) #{acronym} submission (distribution) #{submission.submissionId} has not been parsed." end save_submission_language(submission) diff --git a/helpers/home_helper.rb b/helpers/home_helper.rb index c794dc281..7862c750d 100644 --- a/helpers/home_helper.rb +++ b/helpers/home_helper.rb @@ -5,47 +5,6 @@ module Helpers module HomeHelper - def routes_list - return @navigable_routes if @navigable_routes - - routes = Sinatra::Application.routes['GET'] - navigable_routes = [] - routes.each do |route| - navigable_routes << route[0].to_s.split('?').first - end - @navigable_routes = navigable_routes - navigable_routes - end - - def routes_by_class - { - '/agents' => LinkedData::Models::Agent, - '/annotator' => nil, - '/categories' => LinkedData::Models::Category, - '/groups' => LinkedData::Models::Group, - '/documentation' => nil, - '/mappings' => LinkedData::Models::Mapping, - '/metrics' => LinkedData::Models::Metric, - '/notes' => LinkedData::Models::Note, - '/ontologies' => LinkedData::Models::Ontology, - '/ontologies_full' => LinkedData::Models::Ontology, - '/analytics' => nil, - '/submissions' => LinkedData::Models::OntologySubmission, - '/projects' => LinkedData::Models::Project, - '/property_search' => nil, - '/provisional_classes' => LinkedData::Models::ProvisionalClass, - '/provisional_relations' => LinkedData::Models::ProvisionalRelation, - '/recommender' => nil, - '/replies' => LinkedData::Models::Notes::Reply, - '/reviews' => LinkedData::Models::Review, - '/search' => nil, - '/slices' => LinkedData::Models::Slice, - '/submission_metadata' => nil, - '/ontology_metadata' => nil, - '/users' => LinkedData::Models::User - } - end - def resource_collection_link(cls) resource = @metadata[:cls].name.split("::").last return "" if resource.nil? @@ -71,8 +30,6 @@ def resource_collection_link(cls) "Example: "\ ""\ "/ontologies/NCIT/submissions?display=submissionId,version" - when (routes_list().include? resource_path) == false - "Example: coming soon" else "Resource collection: #{resource_path}" end @@ -96,13 +53,25 @@ def hypermedia_links(cls) def get_metadata_all return @metadata_all_info if @metadata_all_info - - ld_classes = ObjectSpace.each_object(Class).select { |klass| klass < LinkedData::Hypermedia::Resource } info = {} - - ld_classes.each do |cls| - next unless routes_by_class.value?(cls) - + routes_cls = [ + LinkedData::Models::Agent, + LinkedData::Models::Category, + LinkedData::Models::Group, + LinkedData::Models::Mapping, + LinkedData::Models::Metric, + LinkedData::Models::Note, + LinkedData::Models::Ontology, + LinkedData::Models::OntologySubmission, + LinkedData::Models::Project, + LinkedData::Models::ProvisionalClass, + LinkedData::Models::ProvisionalRelation, + LinkedData::Models::Notes::Reply, + LinkedData::Models::Review, + LinkedData::Models::Slice, + LinkedData::Models::User + ] + routes_cls.each do |cls| attributes = if cls.respond_to?(:attributes) (cls.attributes(:all) + cls.hypermedia_settings[:serialize_methods]).uniq else diff --git a/helpers/mod_api_helper.rb b/helpers/mod_api_helper.rb new file mode 100644 index 000000000..479837f81 --- /dev/null +++ b/helpers/mod_api_helper.rb @@ -0,0 +1,119 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module ModApiHelper + + def load_resources_hydra_page(ont, latest_submission, model, attributes, page, size) + check_last_modified_segment(model, [@params["artefactID"]]) + all_count = model.where.in(latest_submission).count + resources = model.where.in(latest_submission).include(attributes).page(page, size).page_count_set(all_count).all + return hydra_page_object(resources.to_a, all_count) + end + + def load_properties_hydra_page(ontology, latest_submission, page, size) + props = ontology.properties(latest_submission) + return hydra_page_object(props.first(size), props.length) + end + + # Resolves a resource by its URI by first fetching its metadata from Solr, + # then using the appropriate model to retrieve the actual data from the ontology or RDF store. + def resolve_resource_by_uri + uri = params['uri'] + ontology_acronym = params['artefactID'] + + error 404, "The uri parameter must be provided via ?uri=" if uri.nil? + + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: ontology_acronym) + check_access(ontology) + + fq = [ + "ontology_t:\"#{ontology_acronym}\"", + "resource_id:\"#{uri}\"" + ] + + conn = SOLR::SolrConnector.new(Goo.search_conf, :ontology_data) + resp = conn.search("*:*", fq: fq, defType: "edismax", start: 0, rows: 1) + doc = resp["response"]["docs"].first + type = doc&.dig("type_t") || doc&.dig("type_txt")&.first + + error 404, "Resource with uri: #{uri} not found" unless doc + + model = model_from_type(type) + + resource = + if model == 'property' + ontology.property(uri, latest_submission) + elsif model + model.find(uri).in(latest_submission).include(model.goo_attrs_to_load(includes_param)).first + end + + return resource + end + + # Maps a resource type string to its corresponding model class. + def model_from_type(type_str) + case type_str + when 'class', 'classes', 'concept', 'concepts', LinkedData::Models::Class.type_uri.to_s, "http://www.w3.org/2004/02/skos/core#Concept" + LinkedData::Models::Class + when 'individuals', 'individual', 'instance', 'instances', LinkedData::Models::Instance.type_uri.to_s + LinkedData::Models::Instance + when 'property', 'properties', LinkedData::Models::AnnotationProperty.type_uri.to_s, LinkedData::Models::ObjectProperty.type_uri.to_s, LinkedData::Models::DatatypeProperty.type_uri.to_s + 'property' + when 'scheme', 'schemes', LinkedData::Models::SKOS::Scheme.type_uri.to_s + LinkedData::Models::SKOS::Scheme + when 'collection', 'collections', LinkedData::Models::SKOS::Collection.type_uri.to_s + LinkedData::Models::SKOS::Collection + when 'label', 'labels', LinkedData::Models::SKOS::Label.type_uri.to_s + LinkedData::Models::SKOS::Label + else + nil + end + end + + # Helper method to find artefact and handle errors + def find_artefact(artefact_id) + artefact = LinkedData::Models::SemanticArtefact.find(artefact_id) + error 404, "Artefact #{artefact_id} not found" if artefact.nil? + artefact + end + + def search_metadata + query = get_query(params) + options = get_ontology_metadata_search_options(params) + page, page_size = page_params + + resp = search(Ontology, query, options) + + result = {} + acronyms_ids = {} + resp.each do |doc| + id = doc["submissionId_i"] + acronym = doc["ontology_acronym_text"] || doc["ontology_t"]&.split('/')&.last + next if acronym.blank? + + old_id = acronyms_ids[acronym].to_i rescue 0 + already_found = (old_id && id && (id <= old_id)) + + next if already_found + + not_restricted = (doc["ontology_viewingRestriction_t"]&.eql?('public') || current_user&.admin?) + user_not_restricted = not_restricted || + Array(doc["ontology_viewingRestriction_txt"]).any? {|u| u.split(' ').last == current_user&.username} || + Array(doc["ontology_acl_txt"]).any? {|u| u.split(' ').last == current_user&.username} + + user_restricted = !user_not_restricted + next if user_restricted + + acronyms_ids[acronym] = id + result[acronym] = LinkedData::Models::SemanticArtefact.read_only(id: "#{LinkedData.settings.id_url_prefix}artefacts/#{acronym}", acronym: acronym, description: doc['description_text'], title: doc['ontology_name_text']) + end + + return hydra_page_object(result.values, result.length) + end + + end + end +end + +helpers Sinatra::Helpers::ModApiHelper diff --git a/helpers/openapi_helper.rb b/helpers/openapi_helper.rb new file mode 100644 index 000000000..3146f8cb8 --- /dev/null +++ b/helpers/openapi_helper.rb @@ -0,0 +1,103 @@ +require 'sinatra/base' +require 'ostruct' + +module Sinatra + module OpenAPIHelper + class OpenAPIDoc + include Sinatra::OpenAPIHelper + Parameter = Struct.new(:name, :in, :required, :type, :description, :default, :schema, keyword_init: true) + Response = Struct.new(:description, :content, keyword_init: true) + + def initialize(tags, summary) + @tags = tags + @summary = summary + @parameters = [] + @responses = {} + end + + def to_hash + { + tags: @tags, + summary: @summary, + parameters: @parameters, + responses: @responses + } + end + + def content(schema, content_type = 'application/json-ld') + { content_type => { schema: schema } } + end + + def response(status, description = nil, content = nil) + @responses[status] = Response.new(description: description, content: content) + end + + def parameter(name, in_: 'query', required: false, type: 'string', description: nil, default: nil, schema: nil) + @parameters << Parameter.new(name: name, in: in_, required: required, type: type, description: description, default: default, schema: schema) + end + + def path_parameter(name, required: true, type: 'string', description: nil, default: nil, schema: nil) + parameter(name, in_: 'path', required: required, type: type, description: description, default: default, schema: schema) + end + + def body_parameter(name, required: true, type: 'object', description: nil, schema: nil) + parameter(name, in_: 'body', required: required, type: type, description: description, schema: schema) + end + end + + def doc(tags = ["default"], summary, &block) + array_tags = tags.is_a?(Array) ? tags : [tags] + doc = OpenAPIDoc.new(array_tags, summary) + doc.instance_eval(&block) + @pending_api_doc = doc.to_hash + end + + def default_params(display: false, pagination: false, query: false) + display_param if display + pagination_params if pagination + query_param if query + end + + def default_responses(success: false, created: false, no_content: false, bad_request: false, unauthorized: false, not_found: false, server_error: false) + response(200, "OK") if success + response(201, "Created") if created + response(204, "No Content") if no_content + response(400, "Bad Request") if bad_request + response(401, "Unauthorized") if unauthorized + response(404, "Not Found") if not_found + response(500, "Internal Server Error") if server_error + end + + def display_param + parameter('display', type: 'string', description: 'Attributes to display', default: '') + end + + def pagination_params + parameter('page', type: 'integer', description: 'Page number', default: '1') + parameter('pagesize', type: 'integer', description: 'Number of items per page', default: '20') + end + + def query_param + parameter('q', type: 'string', description: 'Query text', default: 'plant') + end + + def self.registered(app) + app.before do + @pending_api_doc = nil + end + end + + def route(verb, path, opts = {}, &block) + if @pending_api_doc + @api_docs ||= {} + @api_docs[path.first] ||= {} + @api_docs[path.first][verb.downcase] = @pending_api_doc + @pending_api_doc = nil + end + super(verb, path, opts, &block) + end + end +end + + + diff --git a/helpers/pagination_helper.rb b/helpers/pagination_helper.rb index b91a209ed..0ea988368 100644 --- a/helpers/pagination_helper.rb +++ b/helpers/pagination_helper.rb @@ -32,9 +32,23 @@ def offset_and_limit(page, pagesize) # Return a page object given the total potential results for a call and an array def page_object(array, total_result_count = 0) page, size = page_params - page_obj = LinkedData::Models::Page.new(page, size, total_result_count, array) - page_obj + LinkedData::Models::Page.new(page, size, total_result_count, array) end + + def empty_page + page_object([], 0) + end + + def hydra_page_object(array, total_result_count = 0) + page, size = page_params + LinkedData::Models::HydraPage.new(page, size, total_result_count, array) + end + + def hydra_empty_page + hydra_page_object([], 0) + end + + end end end diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 3805e650d..efb0c1def 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -75,7 +75,7 @@ def get_term_search_query(text, params = {}) if !QUERYLESS_FIELDS_PARAMS.keys.any? { |k| params.key?(k) } || params[EXACT_MATCH_PARAM] == "true" || params[SUGGEST_PARAM] == "true" - raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=]" + raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=] /search?query=[&page=&pagesize=]" else text = '' params['sort'] = 'prefLabelExact asc, submissionAcronym asc' if sort == 'prefLabel' @@ -448,6 +448,183 @@ def validate_params_solr_population(allowed_includes_params) message = "The `include` query string parameter cannot accept #{leftover.join(", ")}, please use only #{allowed_includes_params.join(", ")}" error 400, message if invalid end + + + def get_ontology_metadata_search_options(params) + groups = params.fetch("groups", "").split(',') + categories = params.fetch("hasDomain", "").split(',') + languages = params.fetch("languages", "").split(',') + status = params.fetch("status", "").split(',') + format = params.fetch("hasOntologyLanguage", "").split(',') + is_of_type = params.fetch("isOfType", "").split(',') + has_format = params.fetch("hasFormat", "").split(',') + visibility = params["visibility"] + show_views = params["show_views"] == 'true' + sort = params.fetch("sort", "score desc, ontology_name_sort asc, ontology_acronym_sort asc") + page, page_size = page_params + + fq = [ + 'resource_model:"ontology_submission"', + 'submissionStatus_txt:ERROR_* OR submissionStatus_txt:"RDF" OR submissionStatus_txt:"UPLOADED"', + groups.map { |x| "ontology_group_txt:\"http://data.bioontology.org/groups/#{x.upcase}\"" }.join(' OR '), + categories.map { |x| "ontology_hasDomain_txt:\"http://data.bioontology.org/categories/#{x.upcase}\"" }.join(' OR '), + languages.map { |x| "naturalLanguage_txt:\"#{x.downcase}\"" }.join(' OR '), + ] + + fq << "ontology_viewingRestriction_t:#{visibility}" unless visibility.blank? + fq << "!ontology_viewOf_t:*" unless show_views + + fq << format.map { |x| "hasOntologyLanguage_t:\"http://data.bioontology.org/ontology_formats/#{x}\"" }.join(' OR ') unless format.blank? + + fq << status.map { |x| "status_t:#{x}" }.join(' OR ') unless status.blank? + fq << is_of_type.map { |x| "isOfType_t:#{x}" }.join(' OR ') unless is_of_type.blank? + fq << has_format.map { |x| "hasFormalityLevel_t:#{x}" }.join(' OR ') unless has_format.blank? + + fq.reject!(&:blank?) + + if params[:qf] + qf = params[:qf] + else + qf = [ + "ontologySuggestEdge^25 ontology_acronymSuggestEdge^25 ontology_nameSuggestEdge^15 descriptionSuggestEdge^10 ", # start of the word first + "ontology_t^15 ontology_acronym_text^15 ontology_name_text^10 description_text^5 ", # full word match + "ontologySuggestNgram^2 ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last + ].join(' ') + end + + options = { + fq: fq, + qf: qf, + page: page, + page_size: page_size, + sort: sort + } + options + end + + def get_query(params) + if params[:query].nil? && params[:q].nil? + raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=] /search?query=[&page=&pagesize=]" + end + query = params[:query] || params[:q] + query + end + def build_agent_from_search_result(doc) + affiliations = Array(doc["affiliations_txt"]).map do |aff_txt| + parse_affiliation(aff_txt) + end.compact + + agent_id = doc["id"].split("/").last + usages = LinkedData::Models::Agent + .find(agent_id) + .include(LinkedData::Models::Agent.attributes) + .first + .usages + + LinkedData::Models::Agent.read_only( + id: doc["id"], + agentType: doc["agentType_t"], + name: doc["name_text"], + homepage: doc["homepage_t"], + acronym: doc["acronym_text"], + email: doc["email_text"], + identifiers: doc["identifiers"], + affiliations: affiliations, + creator: doc["creator_t"], + usages: usages + ) + end + def parse_affiliation(aff_txt) + begin + parsed = MultiJson.load(aff_txt) + LinkedData::Models::Agent.read_only( + id: parsed["id"], + name: parsed["name"], + acronym: parsed["acronym"], + email: parsed["email"], + agentType: parsed["agentType"] + ) + rescue MultiJson::ParseError => e + logger.error "Invalid affiliation JSON: #{aff_txt}" + nil + end + end + + def search(model, query, params = {}) + query = query.blank? ? "*" : query + + resp = model.search(query, search_params(**params)) + + total_found = resp["response"]["numFound"] + docs = resp["response"]["docs"] + + page_object(docs, total_found) + end + + def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOperators: "true", page:, page_size:, fl: '*,score', sort:) + { + defType: defType, + fq: fq, + qf: qf, + sort: sort, + start: (page - 1) * page_size, + rows: page_size, + fl: fl, + stopwords: stopwords, + lowercaseOperators: lowercaseOperators, + } + end + + def process_search(params = nil) + params ||= @params + params['q'] ||= params['query'] + params.delete('query') + text = params["q"] + + query = get_term_search_query(text, params) + # puts "Edismax query: #{query}, params: #{params}" + set_page_params(params) + + docs = Array.new + resp = LinkedData::Models::Class.search(query, params) + total_found = resp["response"]["numFound"] + add_matched_fields(resp, Sinatra::Helpers::SearchHelper::MATCH_TYPE_PREFLABEL) + ontology_rank = LinkedData::Models::Ontology.rank + + resp["response"]["docs"].each do |doc| + doc = doc.symbolize_keys + # NCBO-974 + doc[:matchType] = resp["match_types"][doc[:id]] + resource_id = doc[:resource_id] + doc.delete :resource_id + doc[:id] = resource_id + # TODO: The `rescue next` on the following line shouldn't be here + # However, at some point we didn't store the ontologyId in the index + # and these records haven't been cleared out so this is getting skipped + ontology_uri = doc[:ontologyId].sub(/\/submissions\/.*/, "") rescue next + ontology = LinkedData::Models::Ontology.read_only(id: ontology_uri, acronym: doc[:submissionAcronym]) + submission = LinkedData::Models::OntologySubmission.read_only(id: doc[:ontologyId], ontology: ontology) + doc[:submission] = submission + doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 + doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) + + doc = filter_attrs_by_language(doc) + + instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) + docs.push(instance) + end + + unless params['sort'] + if !text.nil? && text[-1] == '*' + docs.sort! { |a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]] } + else + docs.sort! { |a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]] } + end + end + + page_object(docs, total_found) + end + end end end diff --git a/helpers/swagger_ui_helper.rb b/helpers/swagger_ui_helper.rb new file mode 100644 index 000000000..5428a9fb7 --- /dev/null +++ b/helpers/swagger_ui_helper.rb @@ -0,0 +1,48 @@ +require 'json' +require 'sinatra/base' + + +module Sinatra + module SwaggerUI + def generate_openapi_json + { + openapi: '3.0.0', + info: { + title: settings.app_name || 'MOD-API Documentation', + version: settings.api_version || '1.0.0', + description: settings.api_description || 'MOD-API Documentation' + }, + servers: [ + { + url: settings.base_url || '/' + } + ], + tags: [ + { name: 'Artefact', description: 'Get information about semantic artefact(s) (ontologies, terminologies, taxonomies, thesauri, vocabularies, metadata schemas and semantic standards) or their resources.' }, + { name: 'Catalog', description: 'Get information about the semantic artefact catalogue.' }, + { name: 'Record', description: 'Get semantic artefact catalogue records' }, + { name: 'Search', description: 'Search the metadata and catalogue content.' } + ], + paths: generate_paths, + components: { + schemas: settings.respond_to?(:api_schemas) ? settings.api_schemas : {} + } + } + end + + def generate_paths + paths = {} + api_docs = settings.instance_variable_get(:@api_docs) + sorted_paths = api_docs.keys.sort_by do |path| + path.is_a?(Mustermann::Sinatra) ? path.to_s : path + end + + sorted_paths.each do |path| + paths[path] = api_docs[path].transform_keys(&:to_s) + end + paths + end + end +end + +helpers Sinatra::SwaggerUI diff --git a/test/controllers/test_mod_api_controller.rb b/test/controllers/test_mod_api_controller.rb new file mode 100644 index 000000000..4bb637e39 --- /dev/null +++ b/test/controllers/test_mod_api_controller.rb @@ -0,0 +1,317 @@ +require 'webrick' +require_relative '../test_case' + +class TestArtefactsController < TestCase + def before_suite + self.backend_4s_delete + self.class._create_onts + end + + def after_suite + self.backend_4s_delete + end + + def self._create_onts + options = { + ont_count: 2, + submission_count: 2, + submissions_to_process: [1], + process_submission: true, + random_submission_count: false, + acronym: "TST" + } + # this will create 2 ontologies (TST-0, TST-1) with 2 submissions each + @@num_onts_created, @@created_ont_acronyms, @@ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) + @@ontology_0, @@ontology_0_acronym = @@ontologies[0], @@created_ont_acronyms[0] + type = LinkedData::Models::Class.class_rdf_type(@@ontologies[0].latest_submission) + @@ontology_type = type == RDF::OWL[:Class] ? "OWL" : "SKOS" + @@page = 2 + @@pagesize = 1 + @@ontologies[0].latest_submission.index_all(Logger.new($stdout)) + end + + def test_home_controller + get "/" + assert last_response.ok? + catalog_data = MultiJson.load(last_response.body) + + assert catalog_data.key?("links") + assert catalog_data.delete("links").is_a?(Hash) + assert catalog_data.key?("@context") + assert catalog_data.delete("@context").is_a?(Hash) + + expected_data = { + "acronym"=>"OntoPortal", + "title"=>"OntoPortal", + "color"=>"#5499A3", + "description"=>"Welcome to OntoPortal Appliance, your ontology repository for your ontologies", + "logo"=>"https://ontoportal.org/images/logo.png", + "identifier"=>nil, + "status"=>"alpha", + "language"=>["English"], + "accessRights"=>"public", + "license"=>"https://opensource.org/licenses/BSD-2-Clause", + "rightsHolder"=>nil, + "landingPage"=>"http://bioportal.bioontology.org", + "keyword"=>[], + "bibliographicCitation"=>[], + "created"=>nil, + "modified"=>nil, + "contactPoint"=>[], + "creator"=>[], + "contributor"=>[], + "publisher"=>[], + "subject"=>[], + "coverage"=>[], + "createdWith"=>[], + "accrualMethod"=>[], + "accrualPeriodicity"=>[], + "wasGeneratedBy"=>[], + "accessURL"=>"http://data.bioontology.org/", + "numberOfArtefacts"=>2, + "federated_portals"=>[{"name"=>"agroportal", "api"=>"http://data.agroportal.lirmm.fr", "ui"=>"http://agroportal.lirmm.fr", "color"=>"#3cb371"}], + "fundedBy"=>[{"img_src"=>"https://ontoportal.org/images/logo.png", "url"=>"https://ontoportal.org/"}], + "@id"=>"http://data.bioontology.org/", + "@type"=>"https://w3id.org/mod#SemanticArtefactCatalog" + } + + assert_equal expected_data, catalog_data + end + + + def test_all_artefacts + route = '/mod-api/artefacts' + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + artefacts_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, artefacts_page_data) + assert_equal @@num_onts_created, artefacts_page_data["totalItems"] + artefacts_page_data["member"].each do |artefact| + assert @@created_ont_acronyms.include?(artefact["acronym"]) + end + end + + def test_one_artefact + route = "/mod-api/artefacts/#{@@ontology_0_acronym}" + get route + assert last_response.ok? + artefact_data = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, artefact_data["acronym"] + end + + def test_all_distributions + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + dists_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, dists_page_data) + assert_equal 2, dists_page_data["totalItems"] + end + + def test_one_distribution + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions/1" + get route + assert last_response.ok? + dist_data = MultiJson.load(last_response.body) + assert_equal 1, dist_data["distributionId"] + end + + def test_latest_distribution + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions/latest" + get route + assert last_response.ok? + dist_data = MultiJson.load(last_response.body) + assert_equal 2, dist_data["distributionId"] + end + + def test_resources + total_count = total_resources_count + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + resources_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, resources_page_data) + assert_equal total_count, resources_page_data["totalItems"] + end + + def test_one_resource + uri = "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Modular_Component" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{CGI.escape(uri)}" + get route + assert last_response.ok? + resource_data = MultiJson.load(last_response.body) + assert_equal uri, resource_data["@id"] + end + + %w[classes individuals].each do |resource| + define_method("test_#{resource}") do + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + page_data = MultiJson.load(last_response.body) + if @@ontology_type == "OWL" + resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) + validate_hydra_page(route, page_data) + assert_equal resource_count, page_data["totalItems"] + else + validate_hydra_page(route, page_data) + end + end + end + + %w[concepts schemes collections labels].each do |resource| + define_method("test_#{resource}") do + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + page_data = MultiJson.load(last_response.body) + if @@ontology_type == "SKOS" + resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) + validate_hydra_page(route, page_data) + assert_equal resource_count, page_data["totalItems"] + else + validate_hydra_page(route, page_data) + end + end + end + + def test_properties + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/properties" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + properties_page_data = MultiJson.load(last_response.body) + properties_count = @@ontology_0.properties.count + validate_hydra_page(route, properties_page_data) + assert_equal properties_count, properties_page_data["totalItems"] + end + + def test_records + route = "/mod-api/records" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + records_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, records_page_data) + assert_equal @@num_onts_created, records_page_data["totalItems"] + records_page_data["member"].each do |artefact| + assert @@created_ont_acronyms.include?(artefact["acronym"]) + end + end + + def test_one_record + get "/mod-api/records/#{@@ontology_0_acronym}" + assert last_response.ok? + record_data_from_records = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, record_data_from_records["acronym"] + + get "/mod-api/artefacts/#{@@ontology_0_acronym}/record" + assert last_response.ok? + record_data_from_artefact = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, record_data_from_artefact["acronym"] + + assert_equal record_data_from_artefact, record_data_from_records + end + + def test_search_content + route = "/mod-api/search/content" + get "#{route}?query=modular" + assert last_response.ok? + search_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, search_page_data) + end + + def test_search_metadata + route = "/mod-api/search/metadata" + get "#{route}?query=TST-0" + assert last_response.ok? + search_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, search_page_data) + end + + def test_swagger_documentation + get "/openapi.json" + assert last_response.ok? + assert_equal 'application/json', last_response.content_type + + doc = JSON.parse(last_response.body) + + assert_equal '3.0.0', doc['openapi'] + assert_equal 'MOD-API Documentation', doc['info']['title'] + assert_equal '1.0.0', doc['info']['version'] + assert_equal 'Ontoportal MOD-API documentation', doc['info']['description'] + + expected_paths = [ + '/', + '/mod-api/artefacts', + '/mod-api/artefacts/{artefactID}', + '/mod-api/artefacts/{artefactID}/distributions', + '/mod-api/artefacts/{artefactID}/distributions/latest', + '/mod-api/artefacts/{artefactID}/distributions/{distributionID}', + '/mod-api/artefacts/{artefactID}/record', + '/mod-api/artefacts/{artefactID}/resources', + '/mod-api/artefacts/{artefactID}/resources/classes', + '/mod-api/artefacts/{artefactID}/resources/classes/{uri}', + '/mod-api/artefacts/{artefactID}/resources/collections', + '/mod-api/artefacts/{artefactID}/resources/collections/{uri}', + '/mod-api/artefacts/{artefactID}/resources/concepts', + '/mod-api/artefacts/{artefactID}/resources/concepts/{uri}', + '/mod-api/artefacts/{artefactID}/resources/individuals', + '/mod-api/artefacts/{artefactID}/resources/individuals/{uri}', + '/mod-api/artefacts/{artefactID}/resources/labels', + '/mod-api/artefacts/{artefactID}/resources/labels/{uri}', + '/mod-api/artefacts/{artefactID}/resources/properties', + '/mod-api/artefacts/{artefactID}/resources/properties/{uri}', + '/mod-api/artefacts/{artefactID}/resources/schemes', + '/mod-api/artefacts/{artefactID}/resources/schemes/{uri}', + '/mod-api/artefacts/{artefactID}/resources/{uri}', + '/mod-api/records', + '/mod-api/records/{artefactID}', + '/mod-api/search', + '/mod-api/search/content', + '/mod-api/search/metadata' + ] + assert_equal expected_paths.sort, doc['paths'].keys.sort + end + + private + + def validate_hydra_page(route, page_data) + assert page_data.key?('@context') + assert page_data.key?('@id') + assert page_data.key?('@type') + assert page_data.key?("totalItems") + assert page_data.key?('itemsPerPage') + assert page_data.key?('view') + assert page_data['view'].key?('@id') + assert page_data['view'].key?('firstPage') + assert page_data['view'].key?('previousPage') + assert page_data['view'].key?('nextPage') + assert page_data['view'].key?('lastPage') + assert page_data.key?('member') + assert page_data["member"].is_a?(Array) + end + + def total_resources_count + total_count = 0 + resource_model.values.uniq.each do |model| + total_count += model_count(model, @@ontology_0.latest_submission) + end + total_count += @@ontology_0.properties.count + return total_count + end + + def resource_model + { + "classes" => LinkedData::Models::Class, + "concepts" => LinkedData::Models::Class, + "individuals" => LinkedData::Models::Instance, + "schemes" => LinkedData::Models::SKOS::Scheme, + "collections" => LinkedData::Models::SKOS::Collection, + "labels" => LinkedData::Models::SKOS::Label + } + end + + def model_count(model, sub) + model.where.in(sub).count + end + +end \ No newline at end of file diff --git a/views/documentation/metadata.haml b/views/documentation/metadata.haml index 841ac492b..5952da305 100644 --- a/views/documentation/metadata.haml +++ b/views/documentation/metadata.haml @@ -1,4 +1,3 @@ -- return "" unless routes_by_class.value?(@metadata[:cls]) %h3.text-success{id: @metadata[:cls].name.split("::").last}= @metadata[:uri] %div.resource %div.collection_link