From 625af0f462fcafb491c29cbbd1889a8506954f43 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 19 Mar 2025 15:49:23 +0100 Subject: [PATCH 01/23] Feature: use catalog links (#142) * remove links generation logic * remove unused functions --- Gemfile.lock | 51 +++++++++++------------ controllers/home_controller.rb | 21 ---------- helpers/home_helper.rb | 67 +++++++++---------------------- views/documentation/metadata.haml | 1 - 4 files changed, 44 insertions(+), 96 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8398ea56a..3e6009e24 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: b321d73a28b4f60fc5969da7a071b3c19c1a84f3 + revision: 11bf507999651a030cb5550423ce67fb301667f4 branch: development specs: ontologies_linked_data (0.0.1) @@ -103,7 +103,7 @@ GIT GIT remote: https://github.com/sinatra/sinatra.git - revision: c235249abaafa2780b540aca1813dfcf3d17c2dd + revision: cfcc70dee1133690207b5a3dc6000426ec04e250 specs: rack-protection (4.1.1) base64 (>= 0.1.0) @@ -221,23 +221,24 @@ GEM mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - google-cloud-core (1.7.1) + google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.2.1) + google-cloud-env (2.2.2) + base64 (~> 0.2) faraday (>= 1.0, < 3.a) - google-cloud-errors (1.4.0) + google-cloud-errors (1.5.0) google-logging-utils (0.1.0) - google-protobuf (4.29.3) + google-protobuf (4.30.1) bigdecimal rake (>= 13) - google-protobuf (4.29.3-arm64-darwin) + google-protobuf (4.30.1-arm64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.29.3-x86_64-darwin) + google-protobuf (4.30.1-x86_64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.29.3-x86_64-linux) + google-protobuf (4.30.1-x86_64-linux) bigdecimal rake (>= 13) googleapis-common-protos (1.6.0) @@ -246,7 +247,7 @@ GEM grpc (~> 1.41) googleapis-common-protos-types (1.18.0) google-protobuf (>= 3.18, < 5.a) - googleauth (1.13.1) + googleauth (1.14.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.2) google-logging-utils (~> 0.1) @@ -254,16 +255,16 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.70.1) + grpc (1.71.0) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-arm64-darwin) + grpc (1.71.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-x86_64-darwin) + grpc (1.71.0-x86_64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-x86_64-linux) + grpc (1.71.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -278,7 +279,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.10.1) + json (2.10.2) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -306,12 +307,12 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.6.0) + mime-types (3.6.1) logger mime-types-data (~> 3.2015) - mime-types-data (3.2025.0220) + mime-types-data (3.2025.0318) mini_mime (1.1.5) - minitest (5.25.4) + minitest (5.25.5) minitest-fail-fast (0.1.0) minitest (~> 5) minitest-hooks (1.5.2) @@ -351,7 +352,7 @@ GEM net-ssh (7.3.0) netrc (0.11.0) newrelic_rpm (9.17.0) - oj (3.16.9) + oj (3.16.10) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) @@ -372,7 +373,7 @@ GEM public_suffix (6.0.1) raabro (1.4.0) racc (1.8.1) - rack (3.1.10) + rack (3.1.12) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -412,7 +413,7 @@ GEM rexml (~> 3.2) redis (5.4.0) redis-client (>= 0.22.0) - redis-client (0.23.2) + redis-client (0.24.0) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -436,7 +437,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.72.2) + rubocop (1.74.0) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -447,7 +448,7 @@ GEM rubocop-ast (>= 1.38.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.38.0) + rubocop-ast (1.40.0) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) @@ -503,10 +504,10 @@ GEM unicorn-worker-killer (0.4.5) get_process_mem (~> 0) unicorn (>= 4, < 7) - uri (1.0.2) + uri (1.0.3) uuid (2.3.9) macaddr (~> 1.0) - webmock (3.25.0) + webmock (3.25.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 7f3280795..5cc1ace68 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -11,25 +11,6 @@ class HomeController < ApplicationController get do expires 3600, :public last_modified @@root_last_modified ||= Time.now.httpdate - routes = routes_list - - # TODO: delete when ccv will be on production - routes.delete('/ccv') - - routes.delete('/resource_index') if LinkedData.settings.enable_resource_index == false - - routes.delete('/Agents') - - routes_hash = {} - context = {} - - routes.each do |route| - next unless routes_by_class.key?(route) - - route_no_slash = route.gsub('/', '') - context[route_no_slash] = routes_by_class[route].type_uri.to_s if routes_by_class[route].respond_to?(:type_uri) - routes_hash[route_no_slash] = LinkedData.settings.rest_url_prefix + route_no_slash - end catalog_class = LinkedData::Models::SemanticArtefactCatalog catalog = catalog_class.all.first || create_catalog @@ -42,8 +23,6 @@ class HomeController < ApplicationController if catalog.loaded_attributes.include?(:fundedBy) catalog.fundedBy = catalog.fundedBy.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } end - catalog.class.link_to *routes_hash.map { |key, url| LinkedData::Hypermedia::Link.new(key, url, context[key]) } - reply catalog end diff --git a/helpers/home_helper.rb b/helpers/home_helper.rb index c794dc281..7862c750d 100644 --- a/helpers/home_helper.rb +++ b/helpers/home_helper.rb @@ -5,47 +5,6 @@ module Helpers module HomeHelper - def routes_list - return @navigable_routes if @navigable_routes - - routes = Sinatra::Application.routes['GET'] - navigable_routes = [] - routes.each do |route| - navigable_routes << route[0].to_s.split('?').first - end - @navigable_routes = navigable_routes - navigable_routes - end - - def routes_by_class - { - '/agents' => LinkedData::Models::Agent, - '/annotator' => nil, - '/categories' => LinkedData::Models::Category, - '/groups' => LinkedData::Models::Group, - '/documentation' => nil, - '/mappings' => LinkedData::Models::Mapping, - '/metrics' => LinkedData::Models::Metric, - '/notes' => LinkedData::Models::Note, - '/ontologies' => LinkedData::Models::Ontology, - '/ontologies_full' => LinkedData::Models::Ontology, - '/analytics' => nil, - '/submissions' => LinkedData::Models::OntologySubmission, - '/projects' => LinkedData::Models::Project, - '/property_search' => nil, - '/provisional_classes' => LinkedData::Models::ProvisionalClass, - '/provisional_relations' => LinkedData::Models::ProvisionalRelation, - '/recommender' => nil, - '/replies' => LinkedData::Models::Notes::Reply, - '/reviews' => LinkedData::Models::Review, - '/search' => nil, - '/slices' => LinkedData::Models::Slice, - '/submission_metadata' => nil, - '/ontology_metadata' => nil, - '/users' => LinkedData::Models::User - } - end - def resource_collection_link(cls) resource = @metadata[:cls].name.split("::").last return "" if resource.nil? @@ -71,8 +30,6 @@ def resource_collection_link(cls) "Example: "\ ""\ "/ontologies/NCIT/submissions?display=submissionId,version" - when (routes_list().include? resource_path) == false - "Example: coming soon" else "Resource collection: #{resource_path}" end @@ -96,13 +53,25 @@ def hypermedia_links(cls) def get_metadata_all return @metadata_all_info if @metadata_all_info - - ld_classes = ObjectSpace.each_object(Class).select { |klass| klass < LinkedData::Hypermedia::Resource } info = {} - - ld_classes.each do |cls| - next unless routes_by_class.value?(cls) - + routes_cls = [ + LinkedData::Models::Agent, + LinkedData::Models::Category, + LinkedData::Models::Group, + LinkedData::Models::Mapping, + LinkedData::Models::Metric, + LinkedData::Models::Note, + LinkedData::Models::Ontology, + LinkedData::Models::OntologySubmission, + LinkedData::Models::Project, + LinkedData::Models::ProvisionalClass, + LinkedData::Models::ProvisionalRelation, + LinkedData::Models::Notes::Reply, + LinkedData::Models::Review, + LinkedData::Models::Slice, + LinkedData::Models::User + ] + routes_cls.each do |cls| attributes = if cls.respond_to?(:attributes) (cls.attributes(:all) + cls.hypermedia_settings[:serialize_methods]).uniq else diff --git a/views/documentation/metadata.haml b/views/documentation/metadata.haml index 841ac492b..5952da305 100644 --- a/views/documentation/metadata.haml +++ b/views/documentation/metadata.haml @@ -1,4 +1,3 @@ -- return "" unless routes_by_class.value?(@metadata[:cls]) %h3.text-success{id: @metadata[:cls].name.split("::").last}= @metadata[:uri] %div.resource %div.collection_link From e8978f38f6e1a41bf90c25b1f86173cef1e96869 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Fri, 21 Mar 2025 15:38:41 +0100 Subject: [PATCH 02/23] remove class_count in classes route (#143) --- controllers/classes_controller.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/controllers/classes_controller.rb b/controllers/classes_controller.rb index c774f6033..93361aa92 100644 --- a/controllers/classes_controller.rb +++ b/controllers/classes_controller.rb @@ -6,24 +6,20 @@ class ClassesController < ApplicationController get do includes_param_check ont, submission = get_ontology_and_submission - cls_count = submission.class_count(LOGGER) - error 403, "Unable to display classes due to missing metrics for #{submission.id.to_s}. Please contact the administrator." if cls_count < 0 - attributes, page, size, order_by_hash, bring_unmapped_needed = settings_params(LinkedData::Models::Class) + attributes, page, size, order_by_hash = settings_params(LinkedData::Models::Class).first(4) check_last_modified_segment(LinkedData::Models::Class, [ont.acronym]) index = LinkedData::Models::Class.in(submission) if order_by_hash index = index.order_by(order_by_hash) - cls_count = nil # Add index here when, indexing fixed # index_name = 'classes_sort_by_date' # index = index.index_as(index_name) # index = index.with_index(index_name) end - - page_data = index - page_data = page_data.include(attributes).page(page, size).page_count_set(cls_count).all + + page_data = index.include(attributes).page(page, size).all reply page_data end From ad45abcfe4d6bf4e8b6498ea94a71950355f1739 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Fri, 21 Mar 2025 15:40:08 +0100 Subject: [PATCH 03/23] Feature: resources route api implementation (#127) * add /resources route * pagination of resources controllers * regroupe all routes in resources route * fixed address request changes * fixes * make difference between owl and skos * add search routes * fix page and pagesize for schemes, collections and labels * add query param to the search * add tests for artefacts and resources tests --- Gemfile.lock | 8 +- controllers/artefacts.rb | 128 +++++++++++++++ controllers/search_controller.rb | 10 ++ helpers/search_helper.rb | 2 +- test/controllers/test_artefacts_controller.rb | 152 ++++++++++++++++++ 5 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 test/controllers/test_artefacts_controller.rb diff --git a/Gemfile.lock b/Gemfile.lock index 3e6009e24..a33474233 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -145,7 +145,7 @@ GEM airbrussh (1.5.3) sshkit (>= 1.6.1, != 1.7.0) ansi (1.5.0) - ast (2.4.2) + ast (2.4.3) base64 (0.2.0) bcp47_spec (0.2.1) bcrypt (3.1.20) @@ -241,11 +241,11 @@ GEM google-protobuf (4.30.1-x86_64-linux) bigdecimal rake (>= 13) - googleapis-common-protos (1.6.0) + googleapis-common-protos (1.7.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.18.0) + googleapis-common-protos-types (1.19.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.14.0) faraday (>= 1.0, < 3.a) @@ -362,7 +362,7 @@ GEM pandoc-ruby (2.1.10) parallel (1.26.3) parseconfig (1.1.2) - parser (3.3.7.1) + parser (3.3.7.2) ast (~> 2.4.1) racc pony (1.13.1) diff --git a/controllers/artefacts.rb b/controllers/artefacts.rb index 348f43340..9bca2a0c3 100644 --- a/controllers/artefacts.rb +++ b/controllers/artefacts.rb @@ -57,6 +57,134 @@ class ArtefactsController < ApplicationController reply distros.sort {|a,b| b.distributionId.to_i <=> a.distributionId.to_i } end + # Ressources + namespace "/:artefactID/resources" do + get do + ontology, latest_submission = get_ontology_and_latest_submission + _, page, size = settings_params(LinkedData::Models::Class).first(3) + size_per_route = size < 6 ? size : (size / 6).to_i + + resource_types = [ + LinkedData::Models::Class, + LinkedData::Models::Instance, + LinkedData::Models::SKOS::Scheme, + LinkedData::Models::SKOS::Collection, + LinkedData::Models::SKOS::Label + ] + + resources = resource_types.flat_map do |model| + handle_resources_request(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_route).to_a + end + + # add properties because there is no specific model for it + props_page, props_count = handle_properties_request(ontology, latest_submission, page, size_per_route) + resources.concat(props_page.to_a) + + resouces_count = 0 + resource_types.each do |model| + resouces_count += model.where.in(latest_submission).count + end + resouces_count += props_count + + reply Goo::Base::Page.new(page, size, resouces_count, resources) + end + + get '/classes' do + ontology, latest_submission = get_ontology_and_latest_submission + type = LinkedData::Models::Class.class_rdf_type(latest_submission) + attributes, page, size = settings_params(LinkedData::Models::Class).first(3) + + if type == RDF::OWL[:Class] + reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Class, attributes, page, size) + else + reply empty_page(page, size) + end + end + + get '/concepts' do + ontology, latest_submission = get_ontology_and_latest_submission + type = LinkedData::Models::Class.class_rdf_type(latest_submission) + attributes, page, size = settings_params(LinkedData::Models::Class).first(3) + + if type.to_s == "http://www.w3.org/2004/02/skos/core#Concept" + reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Class, attributes, page, size) + else + reply empty_page(page, size) + end + end + + get '/properties' do + ontology, latest_submission = get_ontology_and_latest_submission + _, page, size = settings_params(LinkedData::Models::OntologyProperty).first(3) + props_page, _ = handle_properties_request(ontology, latest_submission, page, size) + reply props_page + end + + get '/individuals' do + ontology, latest_submission = get_ontology_and_latest_submission + type = LinkedData::Models::Class.class_rdf_type(latest_submission) + attributes, page, size = settings_params(LinkedData::Models::Instance).first(3) + if type == RDF::OWL[:Class] + reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Instance, attributes, page, size) + else + reply empty_page(page, size) + end + end + + %w[schemes collections labels].each do |resource_type| + get "/#{resource_type}" do + model_class = case resource_type + when 'schemes' then LinkedData::Models::SKOS::Scheme + when 'collections' then LinkedData::Models::SKOS::Collection + when 'labels' then LinkedData::Models::SKOS::Label + end + + ontology, latest_submission = get_ontology_and_latest_submission + attributes, page, size = settings_params(model_class).first(3) + type = LinkedData::Models::Class.class_rdf_type(latest_submission) + if type.to_s == "http://www.w3.org/2004/02/skos/core#Concept" + reply handle_resources_request(ontology, latest_submission, model_class, attributes, page, size) + else + reply empty_page(page, size) + end + + end + end + + private + + def empty_page(page, size) + Goo::Base::Page.new(page, size, 0, []) + end + + def handle_resources_request(ont, latest_submission, model, attributes, page, size) + check_last_modified_segment(model, [@params["artefactID"]]) + model.where.in(latest_submission).include(attributes).page(page, size).all + end + + def handle_properties_request(ontology, latest_submission, page, size) + props = ontology.properties(latest_submission) + page = Goo::Base::Page.new(page, size, props.length, props.first(size)) + return page, props.length + end + + def get_ontology_and_latest_submission + @ontology ||= Ontology.find(@params["artefactID"]).first + error 404, "You must provide a valid `artefactID` to retrieve an artefact" if @ontology.nil? + + check_last_modified(@ontology) + + @latest_submission ||= @ontology.latest_submission(status: [:RDF]) + error 404, "Artefact #{@params["artefactID"]} distribution not found." if @latest_submission.nil? + + unless @latest_submission.ready?(status: [:RDF]) + error 404, "Artefact #{params["artefactID"]} distribution #{@latest_submission.submissionId} has not been parsed." + end + @latest_submission.bring(ontology: [:acronym]) + return @ontology, @latest_submission + end + end + end end \ No newline at end of file diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 682bd7bf7..bbd6867d9 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -8,6 +8,14 @@ class SearchController < ApplicationController process_search end + get '/content' do + process_search + end + + get '/metadata' do + process_search + end + post do process_search end @@ -208,6 +216,8 @@ def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOper def process_search(params = nil) params ||= @params + params['q'] ||= params['query'] + params.delete('query') text = params["q"] query = get_term_search_query(text, params) diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 3805e650d..7d1b73a78 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -75,7 +75,7 @@ def get_term_search_query(text, params = {}) if !QUERYLESS_FIELDS_PARAMS.keys.any? { |k| params.key?(k) } || params[EXACT_MATCH_PARAM] == "true" || params[SUGGEST_PARAM] == "true" - raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=]" + raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=] /search?query=[&page=&pagesize=]" else text = '' params['sort'] = 'prefLabelExact asc, submissionAcronym asc' if sort == 'prefLabel' diff --git a/test/controllers/test_artefacts_controller.rb b/test/controllers/test_artefacts_controller.rb new file mode 100644 index 000000000..ba3c370a4 --- /dev/null +++ b/test/controllers/test_artefacts_controller.rb @@ -0,0 +1,152 @@ +require 'webrick' +require_relative '../test_case' + +class TestArtefactsController < TestCase + def before_suite + self.backend_4s_delete + self.class._create_onts + end + + def after_suite + self.backend_4s_delete + end + + def self._create_onts + options = { + ont_count: 2, + submission_count: 2, + submissions_to_process: [1], + process_submission: true, + random_submission_count: false, + process_options: {process_rdf: true, extract_metadata: false}, + acronym: "TST" + } + # this will create 2 ontologies (TST-0, TST-1) with 2 submissions each + @@num_onts_created, @@created_ont_acronyms, @@ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options) + @@ontology_0, @@ontology_0_acronym = @@ontologies[0], @@created_ont_acronyms[0] + type = LinkedData::Models::Class.class_rdf_type(@@ontologies[0].latest_submission) + @@ontology_type = type == RDF::OWL[:Class] ? "OWL" : "SKOS" + @@page = 2 + @@pagesize = 1 + end + + + def test_all_artefacts + get "/artefacts?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + artefacts_page_data = MultiJson.load(last_response.body) + validate_page(artefacts_page_data, @@num_onts_created) + artefacts_page_data["collection"].each do |artefact| + assert @@created_ont_acronyms.include?(artefact["acronym"]) + end + end + + def test_one_artefact + get "/artefacts/#{@@ontology_0_acronym}" + assert last_response.ok? + artefact_data = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, artefact_data["acronym"] + end + + def test_all_distributions + get "/artefacts/#{@@ontology_0_acronym}/distributions" + assert last_response.ok? + dists_page_data = MultiJson.load(last_response.body) + assert_equal Array, dists_page_data.class + assert_equal 2, dists_page_data.length + end + + def test_one_distribution + get "/artefacts/#{@@ontology_0_acronym}/distributions/1" + assert last_response.ok? + dist_data = MultiJson.load(last_response.body) + assert_equal 1, dist_data["distributionId"] + end + + def test_latest_distribution + get "/artefacts/#{@@ontology_0_acronym}/distributions/latest" + assert last_response.ok? + dist_data = MultiJson.load(last_response.body) + assert_equal 2, dist_data["distributionId"] + end + + def test_resources + total_count = total_resources_count + get "/artefacts/#{@@ontology_0_acronym}/resources?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + resources_page_data = MultiJson.load(last_response.body) + validate_page(resources_page_data, total_count) + end + + %w[classes individuals].each do |resource| + define_method("test_#{resource}") do + get "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + page_data = MultiJson.load(last_response.body) + if @@ontology_type == "OWL" + resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) + validate_page(page_data, resource_count) + else + validate_page(page_data, 0) + end + end + end + + %w[concepts schemes collections labels].each do |resource| + define_method("test_#{resource}") do + get "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + page_data = MultiJson.load(last_response.body) + if @@ontology_type == "SKOS" + resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) + validate_page(page_data, resource_count) + else + validate_page(page_data, 0) + end + end + end + + def test_properties + get "/artefacts/#{@@ontology_0_acronym}/resources/properties?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + properties_page_data = MultiJson.load(last_response.body) + properties_count = @@ontology_0.properties.count + validate_page(properties_page_data, properties_count) + end + + private + + def validate_page(page_data, resource_count) + assert_equal @@page, page_data["page"] + assert_equal (resource_count/@@pagesize).to_i, page_data["pageCount"] + assert_equal resource_count, page_data["totalCount"] + assert page_data.key?("nextPage") + assert page_data.key?("prevPage") + assert page_data["collection"].is_a?(Array) + end + + def total_resources_count + total_count = 0 + resource_model.values.uniq.each do |model| + total_count += model_count(model, @@ontology_0.latest_submission) + end + total_count += @@ontology_0.properties.count + return total_count + end + + def resource_model + { + "classes" => LinkedData::Models::Class, + "concepts" => LinkedData::Models::Class, + "individuals" => LinkedData::Models::Instance, + "schemes" => LinkedData::Models::SKOS::Scheme, + "collections" => LinkedData::Models::SKOS::Collection, + "labels" => LinkedData::Models::SKOS::Label + } + end + + def model_count(model, sub) + model.where.in(sub).count + end + +end \ No newline at end of file From 609eef024115958586b1b9c0fef634f0af3f889c Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Fri, 25 Apr 2025 18:56:14 +0200 Subject: [PATCH 04/23] Feature: MOD-API implementation iteration 3 (#147) * add /search/metadata * add /resource?uri and /:resourceId to fetch one resource * add recod routes * fix record route * fix * fix search/metadata to return artefacts data * update search metadata and search ontologies * update Gemfile and Gemfile.lock * return SemantiArtefacts objects in search/metadata * address request changes from coderabbitai * address request changes and refactor code * add more tests * remove /type/singular route * use hydra pagination, refactor code and fix tests * fix * apply hydra pagination to search/metadata * use get_ontology_and_submission from the application_helper * safe parse string data to json data * add tests for the home_controller * rescue json parsing error --- Gemfile.lock | 46 +++-- controllers/artefacts.rb | 190 ------------------ controllers/home_controller.rb | 41 ++-- controllers/mod/artefacts_data.rb | 66 ++++++ controllers/mod/artefacts_metadata.rb | 87 ++++++++ controllers/search_controller.rb | 82 +++----- helpers/application_helper.rb | 13 +- helpers/artefact_helper.rb | 85 ++++++++ helpers/pagination_helper.rb | 18 +- helpers/search_helper.rb | 62 ++++++ test/controllers/test_artefacts_controller.rb | 141 ++++++++++--- 11 files changed, 521 insertions(+), 310 deletions(-) delete mode 100644 controllers/artefacts.rb create mode 100644 controllers/mod/artefacts_data.rb create mode 100644 controllers/mod/artefacts_metadata.rb create mode 100644 helpers/artefact_helper.rb diff --git a/Gemfile.lock b/Gemfile.lock index a33474233..cc2aec5e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 11bf507999651a030cb5550423ce67fb301667f4 + revision: 913ff00af28225c00133ba41cc2d51eb08cd90ff branch: development specs: ontologies_linked_data (0.0.1) @@ -103,7 +103,7 @@ GIT GIT remote: https://github.com/sinatra/sinatra.git - revision: cfcc70dee1133690207b5a3dc6000426ec04e250 + revision: 91cfb548c9e50a65324a9ce9e4ea5f10cd897027 specs: rack-protection (4.1.1) base64 (>= 0.1.0) @@ -169,7 +169,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.3.5) - connection_pool (2.5.0) + connection_pool (2.5.2) crack (0.4.5) rexml dante (0.2.0) @@ -181,13 +181,13 @@ GEM ed25519 (1.3.0) et-orbi (1.2.11) tzinfo - faraday (2.12.2) + faraday (2.13.1) faraday-net_http (>= 2.0, < 3.5) json logger faraday-net_http (3.4.0) net-http (>= 0.5.0) - faraday-retry (2.2.1) + faraday-retry (2.3.1) faraday (~> 2.0) ffi (1.15.5) fugit (1.11.1) @@ -229,16 +229,16 @@ GEM faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) google-logging-utils (0.1.0) - google-protobuf (4.30.1) + google-protobuf (4.30.2) bigdecimal rake (>= 13) - google-protobuf (4.30.1-arm64-darwin) + google-protobuf (4.30.2-arm64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.30.1-x86_64-darwin) + google-protobuf (4.30.2-x86_64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.30.1-x86_64-linux) + google-protobuf (4.30.2-x86_64-linux) bigdecimal rake (>= 13) googleapis-common-protos (1.7.0) @@ -279,7 +279,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.10.2) + json (2.11.3) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -298,7 +298,7 @@ GEM libxml-ruby (5.0.3) link_header (0.0.8) lint_roller (1.1.0) - logger (1.6.6) + logger (1.7.0) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -307,10 +307,10 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.6.1) + mime-types (3.6.2) logger mime-types-data (~> 3.2015) - mime-types-data (3.2025.0318) + mime-types-data (3.2025.0422) mini_mime (1.1.5) minitest (5.25.5) minitest-fail-fast (0.1.0) @@ -336,7 +336,7 @@ GEM uri net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.5.6) + net-imap (0.5.7) date net-protocol net-pop (0.1.2) @@ -351,7 +351,7 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.17.0) + newrelic_rpm (9.18.0) oj (3.16.10) bigdecimal (>= 3.0) ostruct (>= 0.2) @@ -360,20 +360,21 @@ GEM os (1.1.4) ostruct (0.6.1) pandoc-ruby (2.1.10) - parallel (1.26.3) + parallel (1.27.0) parseconfig (1.1.2) - parser (3.3.7.2) + parser (3.3.8.0) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) + prism (1.4.0) pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (6.0.1) raabro (1.4.0) racc (1.8.1) - rack (3.1.12) + rack (3.1.13) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -437,7 +438,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.74.0) + rubocop (1.75.3) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -445,11 +446,12 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.44.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.40.0) - parser (>= 3.3.1.0) + rubocop-ast (1.44.1) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) diff --git a/controllers/artefacts.rb b/controllers/artefacts.rb deleted file mode 100644 index 9bca2a0c3..000000000 --- a/controllers/artefacts.rb +++ /dev/null @@ -1,190 +0,0 @@ -class ArtefactsController < ApplicationController - - namespace "/artefacts" do - # Get all Semantic Artefacts - get do - check_last_modified_collection(LinkedData::Models::SemanticArtefact) - attributes, page, pagesize, _, _ = settings_params(LinkedData::Models::SemanticArtefact) - pagesize = 20 if params["pagesize"].nil? - artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) - reply artefacts - end - - # Get one semantic artefact by ID - get "/:artefactID" do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? - check_last_modified(artefact) - artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) - reply artefact - end - - # Display latest distribution - get "/:artefactID/distributions/latest" do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "You must provide a valid artefactID to retrieve an artefact" if artefact.nil? - include_status = params["include_status"] && !params["include_status"].empty? ? params["include_status"].to_sym : :any - latest_distribution = artefact.latest_distribution(status: include_status) - - if latest_distribution - check_last_modified(latest_distribution) - latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - end - reply latest_distribution - end - - # Display a distribution - get '/:artefactID/distributions/:distributionID' do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 422, "Semantic Artefact #{params["artefactID"]} does not exist" unless artefact - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - artefact_distribution = artefact.distribution(params["distributionID"]) - error 404, "Distribuution with #{params['distributionID']} not found" if artefact_distribution.nil? - artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - reply artefact_distribution - end - - # Display a distribution - get '/:artefactID/distributions' do - artefact = LinkedData::Models::SemanticArtefact.find(params["artefactID"]) - error 404, "Semantic Artefact #{params["acronym"]} does not exist" unless artefact - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - options = { - status: (params["include_status"] || "ANY"), - includes: LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) - } - distros = artefact.all_distributions(options) - reply distros.sort {|a,b| b.distributionId.to_i <=> a.distributionId.to_i } - end - - # Ressources - namespace "/:artefactID/resources" do - get do - ontology, latest_submission = get_ontology_and_latest_submission - _, page, size = settings_params(LinkedData::Models::Class).first(3) - size_per_route = size < 6 ? size : (size / 6).to_i - - resource_types = [ - LinkedData::Models::Class, - LinkedData::Models::Instance, - LinkedData::Models::SKOS::Scheme, - LinkedData::Models::SKOS::Collection, - LinkedData::Models::SKOS::Label - ] - - resources = resource_types.flat_map do |model| - handle_resources_request(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_route).to_a - end - - # add properties because there is no specific model for it - props_page, props_count = handle_properties_request(ontology, latest_submission, page, size_per_route) - resources.concat(props_page.to_a) - - resouces_count = 0 - resource_types.each do |model| - resouces_count += model.where.in(latest_submission).count - end - resouces_count += props_count - - reply Goo::Base::Page.new(page, size, resouces_count, resources) - end - - get '/classes' do - ontology, latest_submission = get_ontology_and_latest_submission - type = LinkedData::Models::Class.class_rdf_type(latest_submission) - attributes, page, size = settings_params(LinkedData::Models::Class).first(3) - - if type == RDF::OWL[:Class] - reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Class, attributes, page, size) - else - reply empty_page(page, size) - end - end - - get '/concepts' do - ontology, latest_submission = get_ontology_and_latest_submission - type = LinkedData::Models::Class.class_rdf_type(latest_submission) - attributes, page, size = settings_params(LinkedData::Models::Class).first(3) - - if type.to_s == "http://www.w3.org/2004/02/skos/core#Concept" - reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Class, attributes, page, size) - else - reply empty_page(page, size) - end - end - - get '/properties' do - ontology, latest_submission = get_ontology_and_latest_submission - _, page, size = settings_params(LinkedData::Models::OntologyProperty).first(3) - props_page, _ = handle_properties_request(ontology, latest_submission, page, size) - reply props_page - end - - get '/individuals' do - ontology, latest_submission = get_ontology_and_latest_submission - type = LinkedData::Models::Class.class_rdf_type(latest_submission) - attributes, page, size = settings_params(LinkedData::Models::Instance).first(3) - if type == RDF::OWL[:Class] - reply handle_resources_request(ontology, latest_submission, LinkedData::Models::Instance, attributes, page, size) - else - reply empty_page(page, size) - end - end - - %w[schemes collections labels].each do |resource_type| - get "/#{resource_type}" do - model_class = case resource_type - when 'schemes' then LinkedData::Models::SKOS::Scheme - when 'collections' then LinkedData::Models::SKOS::Collection - when 'labels' then LinkedData::Models::SKOS::Label - end - - ontology, latest_submission = get_ontology_and_latest_submission - attributes, page, size = settings_params(model_class).first(3) - type = LinkedData::Models::Class.class_rdf_type(latest_submission) - if type.to_s == "http://www.w3.org/2004/02/skos/core#Concept" - reply handle_resources_request(ontology, latest_submission, model_class, attributes, page, size) - else - reply empty_page(page, size) - end - - end - end - - private - - def empty_page(page, size) - Goo::Base::Page.new(page, size, 0, []) - end - - def handle_resources_request(ont, latest_submission, model, attributes, page, size) - check_last_modified_segment(model, [@params["artefactID"]]) - model.where.in(latest_submission).include(attributes).page(page, size).all - end - - def handle_properties_request(ontology, latest_submission, page, size) - props = ontology.properties(latest_submission) - page = Goo::Base::Page.new(page, size, props.length, props.first(size)) - return page, props.length - end - - def get_ontology_and_latest_submission - @ontology ||= Ontology.find(@params["artefactID"]).first - error 404, "You must provide a valid `artefactID` to retrieve an artefact" if @ontology.nil? - - check_last_modified(@ontology) - - @latest_submission ||= @ontology.latest_submission(status: [:RDF]) - error 404, "Artefact #{@params["artefactID"]} distribution not found." if @latest_submission.nil? - - unless @latest_submission.ready?(status: [:RDF]) - error 404, "Artefact #{params["artefactID"]} distribution #{@latest_submission.submissionId} has not been parsed." - end - @latest_submission.bring(ontology: [:acronym]) - return @ontology, @latest_submission - end - end - - end - -end \ No newline at end of file diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 5cc1ace68..0f485c86d 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -2,10 +2,6 @@ class HomeController < ApplicationController - CLASS_MAP = { - Property: 'LinkedData::Models::ObjectProperty' - } - namespace '/' do get do @@ -16,13 +12,8 @@ class HomeController < ApplicationController catalog = catalog_class.all.first || create_catalog attributes_to_include = includes_param[0] == :all ? catalog_class.attributes(:all) : catalog_class.goo_attrs_to_load(includes_param) catalog.bring(*attributes_to_include) - if catalog.loaded_attributes.include?(:federated_portals) - catalog.federated_portals = catalog.federated_portals.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - catalog.federated_portals.each { |item| item.delete('apikey') } - end - if catalog.loaded_attributes.include?(:fundedBy) - catalog.fundedBy = catalog.fundedBy.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - end + catalog.federated_portals = safe_parse(catalog.federated_portals) { |item| item.delete('apikey') } if catalog.loaded_attributes.include?(:federated_portals) + catalog.fundedBy = safe_parse(catalog.fundedBy) if catalog.loaded_attributes.include?(:fundedBy) reply catalog end @@ -33,6 +24,7 @@ class HomeController < ApplicationController if catalog.valid? catalog.save status 200 + reply catalog else error 422, catalog.errors end @@ -53,7 +45,7 @@ def create_catalog catalog = nil catalogs = LinkedData::Models::SemanticArtefactCatalog.all if catalogs.nil? || catalogs.empty? - catalog = instance_from_params(LinkedData::Models::SemanticArtefactCatalog, {"test_attr_to_persist" => "test_to_persist"}) + catalog = instance_from_params(LinkedData::Models::SemanticArtefactCatalog, {}) if catalog.valid? catalog.save else @@ -61,8 +53,29 @@ def create_catalog end end catalog - end - + end + + def safe_parse(value) + return nil unless value + + parse_item = ->(item) { + begin + parsed = JSON.parse( + item.gsub(/:(\w+)=>/, '"\1":').gsub('=>', ':').gsub('\"', '"') + ) + yield(parsed) if block_given? + parsed + rescue JSON::ParserError => e + nil + end + } + + if value.is_a?(Array) + value.map { |item| parse_item.call(item) } + else + parse_item.call(value) + end + end end end diff --git a/controllers/mod/artefacts_data.rb b/controllers/mod/artefacts_data.rb new file mode 100644 index 000000000..8fefce2d5 --- /dev/null +++ b/controllers/mod/artefacts_data.rb @@ -0,0 +1,66 @@ +class ArtefactsdataController < ApplicationController + namespace "/artefacts/:artefactID/resources" do + + get do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + _, page, size = settings_params(LinkedData::Models::Class).first(3) + size_per_type = [size / 6, 1].max + + types = [ + LinkedData::Models::Class, + LinkedData::Models::Instance, + LinkedData::Models::SKOS::Scheme, + LinkedData::Models::SKOS::Collection, + LinkedData::Models::SKOS::Label + ] + + total_count = 0 + resources = types.flat_map do |model| + resource_page = load_resources_hydra_page(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_type) + total_count += resource_page.aggregate + resource_page.to_a + end + + props_page = load_properties_hydra_page(ontology, latest_submission, page, size_per_type) + resources.concat(props_page.to_a) + total_count += props_page.aggregate + reply hydra_page_object(resources, total_count) + end + + def self.define_resource_routes(resource_types, expected_type) + resource_types.each do |type| + + get "/#{type}" do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + model_class = (type == 'properties') ? LinkedData::Models::OntologyProperty : model_from_type(type) + attributes, page, size = settings_params(model_class).first(3) + + if type == 'properties' + reply load_properties_hydra_page(ontology, latest_submission, page, size) + else + rdf_type = LinkedData::Models::Class.class_rdf_type(latest_submission) + if rdf_type == expected_type + reply load_resources_hydra_page(ontology, latest_submission, model_class, attributes, page, size) + else + reply hydra_empty_page + end + end + end + + get "/#{type}/:uri" do + reply resolve_resource_by_uri + end + end + end + + define_resource_routes(%w[classes individuals], RDF::OWL[:Class]) + define_resource_routes(%w[concepts schemes collections labels], RDF::Vocab::SKOS[:Concept]) + define_resource_routes(%w[properties], 'properties') + + get '/:uri' do + reply resolve_resource_by_uri + end + end +end diff --git a/controllers/mod/artefacts_metadata.rb b/controllers/mod/artefacts_metadata.rb new file mode 100644 index 000000000..b286f8297 --- /dev/null +++ b/controllers/mod/artefacts_metadata.rb @@ -0,0 +1,87 @@ +class ArtefactsMetadataController < ApplicationController + namespace "/artefacts" do + # Get all Semantic Artefacts + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefact) + attributes, page, pagesize = settings_params(LinkedData::Models::SemanticArtefact).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefact.goo_attrs_to_load([]) if includes_param.first == :all + artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) + reply artefacts + end + + # Get one semantic artefact by ID + get "/:artefactID" do + artefact = find_artefact(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? + check_last_modified(artefact) + artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) + reply artefact + end + + # Get artefact catalog record by ID + get "/:artefactID/record" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + + # Display latest distribution of an artefact + get "/:artefactID/distributions/latest" do + artefact = find_artefact(params["artefactID"]) + include_status = params["include_status"]&.to_sym || :any + latest_distribution = artefact.latest_distribution(status: include_status) + + if latest_distribution + check_last_modified(latest_distribution) + latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + end + reply latest_distribution + end + + # Display a distribution by ID + get '/:artefactID/distributions/:distributionID' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + artefact_distribution = artefact.distribution(params["distributionID"]) + error 404, "Distribution with ID #{params['distributionID']} not found" if artefact_distribution.nil? + artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + reply artefact_distribution + end + + # Display all distributions of an artefact + get '/:artefactID/distributions' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + attributes = LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) if includes_param.first == :all + distros = artefact.all_distributions(attributes, page, pagesize) + reply distros + end + + end + + namespace "/records" do + # Get all Semantic Artefact Catalog Records + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefactCatalogRecord) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load([]) if includes_param.first == :all + records = LinkedData::Models::SemanticArtefactCatalogRecord.all(attributes, page, pagesize) + reply records + end + + # Get a specific record by artefact ID + get "/:artefactID" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + + end +end diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index bbd6867d9..82aa394d0 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -13,7 +13,37 @@ class SearchController < ApplicationController end get '/metadata' do - process_search + query = get_query(params) + options = get_ontology_metadata_search_options(params) + page, page_size = page_params + + resp = search(Ontology, query, options) + + result = {} + acronyms_ids = {} + resp.each do |doc| + id = doc["submissionId_i"] + acronym = doc["ontology_acronym_text"] || doc["ontology_t"]&.split('/')&.last + next if acronym.blank? + + old_id = acronyms_ids[acronym].to_i rescue 0 + already_found = (old_id && id && (id <= old_id)) + + next if already_found + + not_restricted = (doc["ontology_viewingRestriction_t"]&.eql?('public') || current_user&.admin?) + user_not_restricted = not_restricted || + Array(doc["ontology_viewingRestriction_txt"]).any? {|u| u.split(' ').last == current_user&.username} || + Array(doc["ontology_acl_txt"]).any? {|u| u.split(' ').last == current_user&.username} + + user_restricted = !user_not_restricted + next if user_restricted + + acronyms_ids[acronym] = id + result[acronym] = LinkedData::Models::SemanticArtefact.read_only(id: "#{LinkedData.settings.id_url_prefix}artefacts/#{acronym}", acronym: acronym, description: doc['description_text'], title: doc['ontology_name_text']) + end + + reply hydra_page_object(result.values, result.length) end post do @@ -23,54 +53,8 @@ class SearchController < ApplicationController namespace "/ontologies" do get do query = params[:query] || params[:q] - groups = params.fetch("groups", "").split(',') - categories = params.fetch("hasDomain", "").split(',') - languages = params.fetch("languages", "").split(',') - status = params.fetch("status", "").split(',') - format = params.fetch("hasOntologyLanguage", "").split(',') - is_of_type = params.fetch("isOfType", "").split(',') - has_format = params.fetch("hasFormat", "").split(',') - visibility = params["visibility"] - show_views = params["show_views"] == 'true' - sort = params.fetch("sort", "score desc, ontology_name_sort asc, ontology_acronym_sort asc") - page, page_size = page_params - - fq = [ - 'resource_model:"ontology_submission"', - 'submissionStatus_txt:ERROR_* OR submissionStatus_txt:"RDF" OR submissionStatus_txt:"UPLOADED"', - groups.map { |x| "ontology_group_txt:\"http://data.bioontology.org/groups/#{x.upcase}\"" }.join(' OR '), - categories.map { |x| "ontology_hasDomain_txt:\"http://data.bioontology.org/categories/#{x.upcase}\"" }.join(' OR '), - languages.map { |x| "naturalLanguage_txt:\"#{x.downcase}\"" }.join(' OR '), - ] - - fq << "ontology_viewingRestriction_t:#{visibility}" unless visibility.blank? - fq << "!ontology_viewOf_t:*" unless show_views - - fq << format.map { |x| "hasOntologyLanguage_t:\"http://data.bioontology.org/ontology_formats/#{x}\"" }.join(' OR ') unless format.blank? - - fq << status.map { |x| "status_t:#{x}" }.join(' OR ') unless status.blank? - fq << is_of_type.map { |x| "isOfType_t:#{x}" }.join(' OR ') unless is_of_type.blank? - fq << has_format.map { |x| "hasFormalityLevel_t:#{x}" }.join(' OR ') unless has_format.blank? - - fq.reject!(&:blank?) - - if params[:qf] - qf = params[:qf] - else - qf = [ - "ontology_acronymSuggestEdge^25 ontology_nameSuggestEdge^15 descriptionSuggestEdge^10 ", # start of the word first - "ontology_acronym_text^15 ontology_name_text^10 description_text^5 ", # full word match - "ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last - ].join(' ') - end - - page_data = search(Ontology, query, { - fq: fq, - qf: qf, - page: page, - page_size: page_size, - sort: sort - }) + options = get_ontology_metadata_search_options(params) + page_data = search(Ontology, query, options) total_found = page_data.aggregate ontology_rank = LinkedData::Models::Ontology.rank diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index c65541410..856fcf71c 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -390,27 +390,28 @@ def retrieve_latest_submissions(options = {}) latest_submissions end - def get_ontology_and_submission - ont = Ontology.find(@params["ontology"]) + def get_ontology_and_submission(ontology_acronym: nil) + acronym = ontology_acronym || @params["ontology"] + ont = Ontology.find(acronym) .include(:acronym, :administeredBy, :acl, :viewingRestriction) .include(submissions: [:submissionId, submissionStatus: [:code], ontology: [:acronym], metrics: :classes]) .first - error(404, "Ontology '#{@params["ontology"]}' not found.") if ont.nil? + error(404, "Ontology (artefact) '#{acronym}' not found.") if ont.nil? check_access(ont) if LinkedData.settings.enable_security # Security check submission = nil if @params.include? "ontology_submission_id" submission = ont.submission(@params[:ontology_submission_id]) if submission.nil? error 404, - "You must provide an existing submission ID for the #{@params["acronym"]} ontology" + "You must provide an existing submission (distribution) ID for the #{acronym} ontology (artefact)" end else submission = ont.latest_submission(status: [:RDF]) end - error 404, "Ontology #{@params["ontology"]} submission not found." if submission.nil? + error 404, "Ontology (artefact) #{acronym} submission (distribution) not found." if submission.nil? if !submission.ready?(status: [:RDF]) - error 404, "Ontology #{@params["ontology"]} submission #{submission.submissionId} has not been parsed." + error 404, "Ontology (artefact) #{acronym} submission (distribution) #{submission.submissionId} has not been parsed." end save_submission_language(submission) diff --git a/helpers/artefact_helper.rb b/helpers/artefact_helper.rb new file mode 100644 index 000000000..5df0ee5eb --- /dev/null +++ b/helpers/artefact_helper.rb @@ -0,0 +1,85 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module ArtefactHelper + + def load_resources_hydra_page(ont, latest_submission, model, attributes, page, size) + check_last_modified_segment(model, [@params["artefactID"]]) + all_count = model.where.in(latest_submission).count + resources = model.where.in(latest_submission).include(attributes).page(page, size).page_count_set(all_count).all + return hydra_page_object(resources.to_a, all_count) + end + + def load_properties_hydra_page(ontology, latest_submission, page, size) + props = ontology.properties(latest_submission) + return hydra_page_object(props.first(size), props.length) + end + + # Resolves a resource by its URI by first fetching its metadata from Solr, + # then using the appropriate model to retrieve the actual data from the ontology or RDF store. + def resolve_resource_by_uri + uri = params['uri'] + ontology_acronym = params['artefactID'] + + error 404, "The uri parameter must be provided via ?uri=" if uri.nil? + + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: ontology_acronym) + check_access(ontology) + + fq = [ + "ontology_t:\"#{ontology_acronym}\"", + "resource_id:\"#{uri}\"" + ] + + conn = SOLR::SolrConnector.new(Goo.search_conf, :ontology_data) + resp = conn.search("*:*", fq: fq, defType: "edismax", start: 0, rows: 1) + doc = resp["response"]["docs"].first + type = doc&.dig("type_t") || doc&.dig("type_txt")&.first + + error 404, "Resource with uri: #{uri} not found" unless doc + + model = model_from_type(type) + + resource = + if model == 'property' + ontology.property(uri, latest_submission) + elsif model + model.find(uri).in(latest_submission).include(model.goo_attrs_to_load(includes_param)).first + end + + return resource + end + + # Maps a resource type string to its corresponding model class. + def model_from_type(type_str) + case type_str + when 'class', 'classes', 'concept', 'concepts', LinkedData::Models::Class.type_uri.to_s, "http://www.w3.org/2004/02/skos/core#Concept" + LinkedData::Models::Class + when 'individuals', 'individual', 'instance', 'instances', LinkedData::Models::Instance.type_uri.to_s + LinkedData::Models::Instance + when 'property', 'properties', LinkedData::Models::AnnotationProperty.type_uri.to_s, LinkedData::Models::ObjectProperty.type_uri.to_s, LinkedData::Models::DatatypeProperty.type_uri.to_s + 'property' + when 'scheme', 'schemes', LinkedData::Models::SKOS::Scheme.type_uri.to_s + LinkedData::Models::SKOS::Scheme + when 'collection', 'collections', LinkedData::Models::SKOS::Collection.type_uri.to_s + LinkedData::Models::SKOS::Collection + when 'label', 'labels', LinkedData::Models::SKOS::Label.type_uri.to_s + LinkedData::Models::SKOS::Label + else + nil + end + end + + # Helper method to find artefact and handle errors + def find_artefact(artefact_id) + artefact = LinkedData::Models::SemanticArtefact.find(artefact_id) + error 404, "Artefact #{artefact_id} not found" if artefact.nil? + artefact + end + + end + end +end + +helpers Sinatra::Helpers::ArtefactHelper diff --git a/helpers/pagination_helper.rb b/helpers/pagination_helper.rb index b91a209ed..0ea988368 100644 --- a/helpers/pagination_helper.rb +++ b/helpers/pagination_helper.rb @@ -32,9 +32,23 @@ def offset_and_limit(page, pagesize) # Return a page object given the total potential results for a call and an array def page_object(array, total_result_count = 0) page, size = page_params - page_obj = LinkedData::Models::Page.new(page, size, total_result_count, array) - page_obj + LinkedData::Models::Page.new(page, size, total_result_count, array) end + + def empty_page + page_object([], 0) + end + + def hydra_page_object(array, total_result_count = 0) + page, size = page_params + LinkedData::Models::HydraPage.new(page, size, total_result_count, array) + end + + def hydra_empty_page + hydra_page_object([], 0) + end + + end end end diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 7d1b73a78..d88a42f8c 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -448,6 +448,68 @@ def validate_params_solr_population(allowed_includes_params) message = "The `include` query string parameter cannot accept #{leftover.join(", ")}, please use only #{allowed_includes_params.join(", ")}" error 400, message if invalid end + + + def get_ontology_metadata_search_options(params) + groups = params.fetch("groups", "").split(',') + categories = params.fetch("hasDomain", "").split(',') + languages = params.fetch("languages", "").split(',') + status = params.fetch("status", "").split(',') + format = params.fetch("hasOntologyLanguage", "").split(',') + is_of_type = params.fetch("isOfType", "").split(',') + has_format = params.fetch("hasFormat", "").split(',') + visibility = params["visibility"] + show_views = params["show_views"] == 'true' + sort = params.fetch("sort", "score desc, ontology_name_sort asc, ontology_acronym_sort asc") + page, page_size = page_params + + fq = [ + 'resource_model:"ontology_submission"', + 'submissionStatus_txt:ERROR_* OR submissionStatus_txt:"RDF" OR submissionStatus_txt:"UPLOADED"', + groups.map { |x| "ontology_group_txt:\"http://data.bioontology.org/groups/#{x.upcase}\"" }.join(' OR '), + categories.map { |x| "ontology_hasDomain_txt:\"http://data.bioontology.org/categories/#{x.upcase}\"" }.join(' OR '), + languages.map { |x| "naturalLanguage_txt:\"#{x.downcase}\"" }.join(' OR '), + ] + + fq << "ontology_viewingRestriction_t:#{visibility}" unless visibility.blank? + fq << "!ontology_viewOf_t:*" unless show_views + + fq << format.map { |x| "hasOntologyLanguage_t:\"http://data.bioontology.org/ontology_formats/#{x}\"" }.join(' OR ') unless format.blank? + + fq << status.map { |x| "status_t:#{x}" }.join(' OR ') unless status.blank? + fq << is_of_type.map { |x| "isOfType_t:#{x}" }.join(' OR ') unless is_of_type.blank? + fq << has_format.map { |x| "hasFormalityLevel_t:#{x}" }.join(' OR ') unless has_format.blank? + + fq.reject!(&:blank?) + + if params[:qf] + qf = params[:qf] + else + qf = [ + "ontologySuggestEdge^25 ontology_acronymSuggestEdge^25 ontology_nameSuggestEdge^15 descriptionSuggestEdge^10 ", # start of the word first + "ontology_t^15 ontology_acronym_text^15 ontology_name_text^10 description_text^5 ", # full word match + "ontologySuggestNgram^2 ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last + ].join(' ') + end + + options = { + fq: fq, + qf: qf, + page: page, + page_size: page_size, + sort: sort + } + options + end + + def get_query(params) + if params[:query].nil? && params[:q].nil? + raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=] /search?query=[&page=&pagesize=]" + end + query = params[:query] || params[:q] + query + end + end end end diff --git a/test/controllers/test_artefacts_controller.rb b/test/controllers/test_artefacts_controller.rb index ba3c370a4..7269c2c7f 100644 --- a/test/controllers/test_artefacts_controller.rb +++ b/test/controllers/test_artefacts_controller.rb @@ -30,99 +30,186 @@ def self._create_onts @@pagesize = 1 end + def test_home_controller + get "/" + assert last_response.ok? + catalog_data = MultiJson.load(last_response.body) + + assert catalog_data.key?("links") + assert catalog_data.delete("links").is_a?(Hash) + assert catalog_data.key?("@context") + assert catalog_data.delete("@context").is_a?(Hash) + + expected_data = { + "acronym"=>"OntoPortal", + "title"=>"OntoPortal", + "color"=>"#5499A3", + "description"=>"Welcome to OntoPortal Appliance, your ontology repository for your ontologies", + "logo"=>"https://ontoportal.org/images/logo.png", + "identifier"=>nil, + "status"=>"alpha", + "language"=>["English"], + "accessRights"=>"public", + "license"=>"https://opensource.org/licenses/BSD-2-Clause", + "rightsHolder"=>nil, + "landingPage"=>"http://bioportal.bioontology.org", + "keyword"=>[], + "bibliographicCitation"=>[], + "created"=>nil, + "modified"=>nil, + "contactPoint"=>[], + "creator"=>[], + "contributor"=>[], + "publisher"=>[], + "subject"=>[], + "coverage"=>[], + "createdWith"=>[], + "accrualMethod"=>[], + "accrualPeriodicity"=>[], + "wasGeneratedBy"=>[], + "accessURL"=>"http://data.bioontology.org/", + "numberOfArtefacts"=>2, + "federated_portals"=>[{"name"=>"agroportal", "api"=>"http://data.agroportal.lirmm.fr", "ui"=>"http://agroportal.lirmm.fr", "color"=>"#3cb371"}], + "fundedBy"=>[{"img_src"=>"https://ontoportal.org/images/logo.png", "url"=>"https://ontoportal.org/"}], + "@id"=>"http://data.bioontology.org/", + "@type"=>"https://w3id.org/mod#SemanticArtefactCatalog" + } + + assert_equal expected_data, catalog_data + end + def test_all_artefacts - get "/artefacts?page=#{@@page}&pagesize=#{@@pagesize}" + route = '/artefacts' + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? artefacts_page_data = MultiJson.load(last_response.body) - validate_page(artefacts_page_data, @@num_onts_created) - artefacts_page_data["collection"].each do |artefact| + validate_hydra_page(route, artefacts_page_data, @@num_onts_created) + artefacts_page_data["member"].each do |artefact| assert @@created_ont_acronyms.include?(artefact["acronym"]) end end def test_one_artefact - get "/artefacts/#{@@ontology_0_acronym}" + route = "/artefacts/#{@@ontology_0_acronym}" + get route assert last_response.ok? artefact_data = MultiJson.load(last_response.body) assert_equal @@ontology_0_acronym, artefact_data["acronym"] end def test_all_distributions - get "/artefacts/#{@@ontology_0_acronym}/distributions" + route = "/artefacts/#{@@ontology_0_acronym}/distributions" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? dists_page_data = MultiJson.load(last_response.body) - assert_equal Array, dists_page_data.class - assert_equal 2, dists_page_data.length + validate_hydra_page(route, dists_page_data, 2) end def test_one_distribution - get "/artefacts/#{@@ontology_0_acronym}/distributions/1" + route = "/artefacts/#{@@ontology_0_acronym}/distributions/1" + get route assert last_response.ok? dist_data = MultiJson.load(last_response.body) assert_equal 1, dist_data["distributionId"] end def test_latest_distribution - get "/artefacts/#{@@ontology_0_acronym}/distributions/latest" + route = "/artefacts/#{@@ontology_0_acronym}/distributions/latest" + get route assert last_response.ok? dist_data = MultiJson.load(last_response.body) - assert_equal 2, dist_data["distributionId"] + assert_equal 1, dist_data["distributionId"] end def test_resources total_count = total_resources_count - get "/artefacts/#{@@ontology_0_acronym}/resources?page=#{@@page}&pagesize=#{@@pagesize}" + route = "/artefacts/#{@@ontology_0_acronym}/resources" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? resources_page_data = MultiJson.load(last_response.body) - validate_page(resources_page_data, total_count) + validate_hydra_page(route, resources_page_data, total_count) end %w[classes individuals].each do |resource| define_method("test_#{resource}") do - get "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}?page=#{@@page}&pagesize=#{@@pagesize}" + route = "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? page_data = MultiJson.load(last_response.body) if @@ontology_type == "OWL" resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) - validate_page(page_data, resource_count) + validate_hydra_page(route, page_data, resource_count) else - validate_page(page_data, 0) + validate_hydra_page(route, page_data, 0) end end end %w[concepts schemes collections labels].each do |resource| define_method("test_#{resource}") do - get "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}?page=#{@@page}&pagesize=#{@@pagesize}" + route = "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? page_data = MultiJson.load(last_response.body) if @@ontology_type == "SKOS" resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) - validate_page(page_data, resource_count) + validate_hydra_page(route, page_data, resource_count) else - validate_page(page_data, 0) + validate_hydra_page(route, page_data, 0) end end end def test_properties - get "/artefacts/#{@@ontology_0_acronym}/resources/properties?page=#{@@page}&pagesize=#{@@pagesize}" + route = "/artefacts/#{@@ontology_0_acronym}/resources/properties" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? properties_page_data = MultiJson.load(last_response.body) properties_count = @@ontology_0.properties.count - validate_page(properties_page_data, properties_count) + validate_hydra_page(route, properties_page_data, properties_count) + end + + def test_records + route = "/records" + get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" + assert last_response.ok? + records_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, records_page_data, @@num_onts_created) + records_page_data["member"].each do |artefact| + assert @@created_ont_acronyms.include?(artefact["acronym"]) + end + end + + def test_one_record + get "/records/#{@@ontology_0_acronym}" + assert last_response.ok? + record_data_from_records = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, record_data_from_records["acronym"] + + get "/artefacts/#{@@ontology_0_acronym}/record" + assert last_response.ok? + record_data_from_artefact = MultiJson.load(last_response.body) + assert_equal @@ontology_0_acronym, record_data_from_artefact["acronym"] + + assert_equal record_data_from_artefact, record_data_from_records end private - def validate_page(page_data, resource_count) - assert_equal @@page, page_data["page"] - assert_equal (resource_count/@@pagesize).to_i, page_data["pageCount"] - assert_equal resource_count, page_data["totalCount"] - assert page_data.key?("nextPage") - assert page_data.key?("prevPage") - assert page_data["collection"].is_a?(Array) + def validate_hydra_page(route, page_data, resource_count) + assert page_data.key?('@context') + assert_equal "#{LinkedData.settings.rest_url_prefix.chomp("/")}#{route}", page_data['@id'] + assert_equal 'hydra:Collection', page_data['@type'] + assert_equal resource_count, page_data["totalItems"] + assert page_data.key?('itemsPerPage') + assert page_data.key?('view') + assert_equal "#{LinkedData.settings.rest_url_prefix.chomp("/")}#{route}?page=#{@@page}&pagesize=#{@@pagesize}", page_data['view']['@id'] + assert page_data['view'].key?('firstPage') + assert page_data['view'].key?('previousPage') + assert page_data['view'].key?('nextPage') + assert page_data['view'].key?('lastPage') + assert page_data["member"].is_a?(Array) end def total_resources_count From 3664fdcadb1aa782e90c30e120836d548a00c1d9 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Fri, 16 May 2025 21:08:18 +0200 Subject: [PATCH 05/23] Feature: Add Sentry monitoring to api (#151) * replace newrelic with sentry * update bin ontoportal to use virtuoso by default * use production sentry key * fix docker testing * use and old version of ontologies linked data to fix tests --- .github/workflows/ruby-unit-tests.yml | 1 + Capfile | 1 - Gemfile | 9 +- Gemfile.lock | 144 ++++++++++++--------- app.rb | 15 ++- bin/ontoportal | 4 +- config/environments/config.rb.sample | 1 + docker-compose.yml | 16 +-- test/controllers/test_agents_controller.rb | 56 ++++---- 9 files changed, 137 insertions(+), 110 deletions(-) diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 39d28a8c5..75ad06930 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -28,6 +28,7 @@ jobs: bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Run unit tests run: | + cp .env.sample .env ci_env=`bash <(curl -s https://codecov.io/env)` GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} diff --git a/Capfile b/Capfile index 7ecc995cd..95799ba1b 100644 --- a/Capfile +++ b/Capfile @@ -22,6 +22,5 @@ require 'capistrano/bundler' # require 'capistrano/rails/assets' # require 'capistrano/rails/migrations' require 'capistrano/locally' -require 'new_relic/recipes' # announce deployments in NewRelic # Loads custom tasks from `lib/capistrano/tasks' if you have any defined. Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r } diff --git a/Gemfile b/Gemfile index 4c3e56380..789aacaef 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ end gem 'request_store' gem 'parallel' -gem 'google-protobuf' +gem 'google-protobuf', '~> 4.30.0' gem 'net-ftp' gem 'json-ld', '~> 3.2.0' gem 'rdf-raptor', github:'ruby-rdf/rdf-raptor', ref: '6392ceabf71c3233b0f7f0172f662bd4a22cd534' # use version 3.3.0 when available @@ -38,7 +38,7 @@ gem 'redis' gem 'redis-store' # Monitoring -gem 'newrelic_rpm', group: [:default, :deployment] +gem "sentry-ruby", "~> 5.24" # HTTP server gem 'unicorn' @@ -53,8 +53,8 @@ gem 'pandoc-ruby' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ontoportal-lirmm/ncbo_ontology_recommender.git', branch: 'development' -gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'development' -gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', ref: '54ca6e05268746412fe4021207629d1b5c69700c' +gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development' group :development do @@ -91,3 +91,4 @@ group :test do gem 'webmock' gem 'webrick' end + diff --git a/Gemfile.lock b/Gemfile.lock index 8398ea56a..768e629d6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: e48a2d13a65cc2dd1c12d116cfc9da9061106861 - branch: development + revision: ceacfbbccca94c82829071232d95f80c74eafb44 + branch: master specs: goo (0.0.2) addressable (~> 2.8) @@ -18,7 +18,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git - revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + revision: aeb0222400f1b423cb865545c41233d2cbd82bfc branch: development specs: ncbo_annotator (0.0.1) @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: cc4cd9218db7181c4843772631b7f3a96c74a4aa + revision: 3cf75117e8023115cfdec1c8d5a3d78f273d19db branch: master specs: ncbo_cron (0.0.1) @@ -57,8 +57,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: b321d73a28b4f60fc5969da7a071b3c19c1a84f3 - branch: development + revision: 54ca6e05268746412fe4021207629d1b5c69700c + ref: 54ca6e05268746412fe4021207629d1b5c69700c specs: ontologies_linked_data (0.0.1) activesupport @@ -103,7 +103,7 @@ GIT GIT remote: https://github.com/sinatra/sinatra.git - revision: c235249abaafa2780b540aca1813dfcf3d17c2dd + revision: 91cfb548c9e50a65324a9ce9e4ea5f10cd897027 specs: rack-protection (4.1.1) base64 (>= 0.1.0) @@ -145,7 +145,7 @@ GEM airbrussh (1.5.3) sshkit (>= 1.6.1, != 1.7.0) ansi (1.5.0) - ast (2.4.2) + ast (2.4.3) base64 (0.2.0) bcp47_spec (0.2.1) bcrypt (3.1.20) @@ -169,7 +169,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.3.5) - connection_pool (2.5.0) + connection_pool (2.5.3) crack (0.4.5) rexml dante (0.2.0) @@ -178,22 +178,22 @@ GEM docile (1.4.1) domain_name (0.6.20240107) drb (2.2.1) - ed25519 (1.3.0) + ed25519 (1.4.0) et-orbi (1.2.11) tzinfo - faraday (2.12.2) + faraday (2.13.1) faraday-net_http (>= 2.0, < 3.5) json logger faraday-net_http (3.4.0) net-http (>= 0.5.0) - faraday-retry (2.2.1) + faraday-retry (2.3.1) faraday (~> 2.0) ffi (1.15.5) fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) - gapic-common (0.25.0) + gapic-common (1.0.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) google-cloud-env (~> 2.2) @@ -205,15 +205,15 @@ GEM grpc (~> 1.66) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.7.0) + google-analytics-data (0.7.1) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.16.0) - gapic-common (>= 0.25.0, < 2.a) + google-analytics-data-v1beta (0.17.0) + gapic-common (~> 1.0) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.16.0) + google-apis-analytics_v3 (0.17.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.16.0) + google-apis-core (0.17.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -221,32 +221,39 @@ GEM mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - google-cloud-core (1.7.1) + google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.2.1) + google-cloud-env (2.3.0) + base64 (~> 0.2) faraday (>= 1.0, < 3.a) - google-cloud-errors (1.4.0) - google-logging-utils (0.1.0) - google-protobuf (4.29.3) + google-cloud-errors (1.5.0) + google-logging-utils (0.2.0) + google-protobuf (4.30.2) + bigdecimal + rake (>= 13) + google-protobuf (4.30.2-aarch64-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.30.2-arm64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.29.3-arm64-darwin) + google-protobuf (4.30.2-x86-linux) bigdecimal rake (>= 13) - google-protobuf (4.29.3-x86_64-darwin) + google-protobuf (4.30.2-x86_64-darwin) bigdecimal rake (>= 13) - google-protobuf (4.29.3-x86_64-linux) + google-protobuf (4.30.2-x86_64-linux) bigdecimal rake (>= 13) - googleapis-common-protos (1.6.0) + googleapis-common-protos (1.8.0) google-protobuf (>= 3.18, < 5.a) - googleapis-common-protos-types (~> 1.7) + googleapis-common-protos-types (~> 1.20) grpc (~> 1.41) - googleapis-common-protos-types (1.18.0) + googleapis-common-protos-types (1.20.0) google-protobuf (>= 3.18, < 5.a) - googleauth (1.13.1) + googleauth (1.14.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.2) google-logging-utils (~> 0.1) @@ -254,16 +261,22 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.70.1) + grpc (1.72.0) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-arm64-darwin) + grpc (1.72.0-aarch64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-x86_64-darwin) + grpc (1.72.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.70.1-x86_64-linux) + grpc (1.72.0-x86-linux) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) + grpc (1.72.0-x86_64-darwin) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) + grpc (1.72.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -278,7 +291,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.10.1) + json (2.12.0) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -293,11 +306,11 @@ GEM jwt (2.10.1) base64 kgio (2.11.4) - language_server-protocol (3.17.0.4) - libxml-ruby (5.0.3) + language_server-protocol (3.17.0.5) + libxml-ruby (5.0.4) link_header (0.0.8) lint_roller (1.1.0) - logger (1.6.6) + logger (1.7.0) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -306,12 +319,12 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.6.0) + mime-types (3.7.0) logger - mime-types-data (~> 3.2015) - mime-types-data (3.2025.0220) + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2025.0514) mini_mime (1.1.5) - minitest (5.25.4) + minitest (5.25.5) minitest-fail-fast (0.1.0) minitest (~> 5) minitest-hooks (1.5.2) @@ -335,7 +348,7 @@ GEM uri net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.5.6) + net-imap (0.5.8) date net-protocol net-pop (0.1.2) @@ -350,8 +363,7 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.17.0) - oj (3.16.9) + oj (3.16.10) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) @@ -359,20 +371,21 @@ GEM os (1.1.4) ostruct (0.6.1) pandoc-ruby (2.1.10) - parallel (1.26.3) + parallel (1.27.0) parseconfig (1.1.2) - parser (3.3.7.1) + parser (3.3.8.0) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) + prism (1.4.0) pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.10) + rack (3.1.14) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -381,11 +394,12 @@ GEM rack (>= 0.4) rack-contrib (2.5.0) rack (< 4) - rack-cors (2.0.2) - rack (>= 2.0.0) + rack-cors (3.0.0) + logger + rack (>= 3.0.14) rack-mini-profiler (3.3.1) rack (>= 1.2.0) - rack-session (2.1.0) + rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) @@ -412,7 +426,7 @@ GEM rexml (~> 3.2) redis (5.4.0) redis-client (>= 0.22.0) - redis-client (0.23.2) + redis-client (0.24.0) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) @@ -436,7 +450,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.72.2) + rubocop (1.75.6) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -444,11 +458,12 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.44.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.38.0) - parser (>= 3.3.1.0) + rubocop-ast (1.44.1) + parser (>= 3.3.7.2) + prism (~> 1.4) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) ruby2_keywords (0.0.5) @@ -456,7 +471,10 @@ GEM rufus-scheduler (3.9.2) fugit (~> 1.1, >= 1.11.1) securerandom (0.4.1) - signet (0.19.0) + sentry-ruby (5.24.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -503,18 +521,20 @@ GEM unicorn-worker-killer (0.4.5) get_process_mem (~> 0) unicorn (>= 4, < 7) - uri (1.0.2) + uri (1.0.3) uuid (2.3.9) macaddr (~> 1.0) - webmock (3.25.0) + webmock (3.25.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.9.1) PLATFORMS + aarch64-linux arm64-darwin ruby + x86-linux x86_64-darwin x86_64-linux @@ -530,7 +550,7 @@ DEPENDENCIES ed25519 (>= 1.2, < 2.0) ffi (~> 1.15.0) goo! - google-protobuf + google-protobuf (~> 4.30.0) haml (~> 5.2.2) json-ld (~> 3.2.0) json-schema @@ -544,7 +564,6 @@ DEPENDENCIES ncbo_cron! ncbo_ontology_recommender! net-ftp - newrelic_rpm oj ontologies_linked_data! pandoc-ruby @@ -569,6 +588,7 @@ DEPENDENCIES request_store rexml rubocop + sentry-ruby (~> 5.24) shotgun! simplecov simplecov-cobertura diff --git a/app.rb b/app.rb index be90bd342..d7d4c3150 100644 --- a/app.rb +++ b/app.rb @@ -135,8 +135,19 @@ require_relative 'config/unicorn_workerkiller' end -# Add New Relic last to allow Rack middleware instrumentation -require 'newrelic_rpm' +if $SENTRY_DSN + require 'sentry-ruby' + Sentry.init do |config| + config.dsn = $SENTRY_DSN + + # Add data like request headers and IP for users, + # see https://docs.sentry.io/platforms/ruby/data-management/data-collected/ for more info + config.send_default_pii = true + end + + use Rack::RewindableInput::Middleware + use Sentry::Rack::CaptureExceptions +end # Initialize the app require_relative 'init' diff --git a/bin/ontoportal b/bin/ontoportal index 9bf8cf5ba..f9c7ca18f 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -78,7 +78,7 @@ build_docker_run_cmd() { local goo_path="$3" local sparql_client_path="$4" - local docker_run_cmd="docker compose -p ontoportal_docker run --rm -it --name api-service" + local docker_run_cmd="docker compose --profile vo -p ontoportal_docker run --rm -it --name api-service" local bash_cmd="" # Conditionally add bind mounts only if the paths are not empty @@ -108,7 +108,7 @@ provision() { source .env echo "[+] Cleaning volumes" - docker compose -f docker-compose.yml --profile 4store down --volumes >/dev/null 2>&1 + docker compose -f docker-compose.yml --profile vo down --volumes >/dev/null 2>&1 docker compose -p ontoportal_docker down --volumes >/dev/null 2>&1 commands=( diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 0eabcee81..0c6c874ec 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -22,6 +22,7 @@ REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PR SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr" SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr" +$SENTRY_DSN = ENV.include?("SENTRY_DSN") ? ENV["SENTRY_DSN"] : nil begin # For prefLabel extract main_lang first, or anything if no main found. # For other properties only properties with a lang that is included in main_lang are used diff --git a/docker-compose.yml b/docker-compose.yml index 564fc8d2d..1cc57b7b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,9 @@ x-app: &app REDIS_PORT: 6379 SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr - GOO_BACKEND_NAME: 4store - GOO_PORT: 9000 - GOO_HOST: 4store-ut + GOO_BACKEND_NAME: virtuoso + GOO_PORT: 8890 + GOO_HOST: virtuoso-ut MGREP_HOST: mgrep-ut MGREP_PORT: 55555 REPOSITORY_FOLDER: /srv/ontoportal/data/repository @@ -32,8 +32,6 @@ services: <<: *env BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle - profiles: - - 4store depends_on: solr-ut: condition: service_healthy @@ -41,7 +39,7 @@ services: condition: service_healthy mgrep-ut: condition: service_started - 4store-ut: + virtuoso-ut: condition: service_started ncbo_cron: condition: service_started @@ -61,8 +59,6 @@ services: <<: *env BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_cron/.bundle command: "bundle exec bin/ncbo_cron" - profiles: - - 4store volumes: - app_cron:/srv/ontoportal/ncbo_cron - repository:/srv/ontoportal/data/repository @@ -77,7 +73,7 @@ services: condition: service_healthy mgrep-ut: condition: service_started - 4store-ut: + virtuoso-ut: condition: service_started @@ -159,8 +155,6 @@ services: ports: - 1111:1111 - 8890:8890 - profiles: - - vo healthcheck: test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] start_period: 10s diff --git a/test/controllers/test_agents_controller.rb b/test/controllers/test_agents_controller.rb index 658ef38b9..75a2783f7 100644 --- a/test/controllers/test_agents_controller.rb +++ b/test/controllers/test_agents_controller.rb @@ -1,5 +1,5 @@ require_relative '../test_case' -require "multi_json" +require 'multi_json' class TestAgentsController < TestCase @@ -33,14 +33,14 @@ def test_all_agents created_agents = MultiJson.load(last_response.body) @agents.each do |agent| - created_agent = created_agents["collection"].select{|x| x["name"].eql?(agent[:name])}.first + created_agent = created_agents['collection'].select{|x| x['name'].eql?(agent[:name])}.first refute_nil created_agent - refute_nil created_agent["usages"] - assert_equal agent[:name], created_agent["name"] - assert_equal agent[:identifiers].size, created_agent["identifiers"].size - assert_equal agent[:identifiers].map{|x| x[:notation]}.sort, created_agent["identifiers"].map{|x| x['notation']}.sort - assert_equal agent[:affiliations].size, created_agent["affiliations"].size - assert_equal agent[:affiliations].map{|x| x["name"]}.sort, created_agent["affiliations"].map{|x| x['name']}.sort + refute_nil created_agent['usages'] + assert_equal agent[:name], created_agent['name'] + assert_equal agent[:identifiers].size, created_agent['identifiers'].size + assert_equal agent[:identifiers].map{|x| x[:notation]}.sort, created_agent['identifiers'].map{|x| x['notation']}.sort + assert_equal agent[:affiliations].size, created_agent['affiliations'].size + assert_equal agent[:affiliations].map{|x| x['name']}.sort, created_agent['affiliations'].map{|x| x['name']}.sort end end @@ -51,7 +51,7 @@ def test_single_agent get "/agents/#{agent_obj.id.to_s.split('/').last}" assert last_response.ok? agent_found = MultiJson.load(last_response.body) - assert_equal agent_obj.id.to_s, agent_found["id"] + assert_equal agent_obj.id.to_s, agent_found['id'] end end @@ -76,7 +76,7 @@ def test_create_new_agent ## Create Agent of type Person with an extent affiliations agent = @test_agents[6] - agent[:affiliations] = created_agent["affiliations"] + agent[:affiliations] = created_agent['affiliations'] _test_agent_creation(agent) ## Create Agent of type Person with no extent affiliations @@ -93,7 +93,7 @@ def test_new_agent_no_valid agents_tmp = [ _agent_data(type: 'organization'), _agent_data(type: 'person'), _agent_data(type: 'person')] agent = agents_tmp.last agent[:affiliations] = [agents_tmp[0].stringify_keys, agents_tmp[1].stringify_keys] - post "/agents", MultiJson.dump(agent), "CONTENT_TYPE" => "application/json" + post '/agents', MultiJson.dump(agent), 'CONTENT_TYPE' => 'application/json' assert last_response.status == 400 end @@ -139,21 +139,21 @@ def test_update_patch_agent affiliations: new_affiliations } - patch "/agents/#{agent.id.split('/').last}", MultiJson.dump(new_values), "CONTENT_TYPE" => "application/json" + patch "/agents/#{agent.id.split('/').last}", MultiJson.dump(new_values), 'CONTENT_TYPE' => 'application/json' assert last_response.status == 204 get "/agents/#{agent.id.split('/').last}" new_agent = MultiJson.load(last_response.body) - assert_equal 'new name ', new_agent["name"] + assert_equal 'new name ', new_agent['name'] - assert_equal new_identifiers.size, new_agent["identifiers"].size - assert_equal new_identifiers[0][:schemaAgency], new_agent["identifiers"].select{|x| x["id"].eql?(agent.identifiers[0].id.to_s)}.first["schemaAgency"] - assert_equal agent.identifiers[1].schemaAgency, new_agent["identifiers"].select{|x| x["id"].eql?(agent.identifiers[1].id.to_s)}.first["schemaAgency"] + assert_equal new_identifiers.size, new_agent['identifiers'].size + assert_equal new_identifiers[0][:schemaAgency], new_agent['identifiers'].select{|x| x['id'].eql?(agent.identifiers[0].id.to_s)}.first['schemaAgency'] + assert_equal agent.identifiers[1].schemaAgency, new_agent['identifiers'].select{|x| x['id'].eql?(agent.identifiers[1].id.to_s)}.first['schemaAgency'] - assert_equal new_affiliations.size, new_agent["affiliations"].size - assert_equal new_affiliations[0][:name], new_agent["affiliations"].select{|x| x["id"].eql?(agent.affiliations[0].id.to_s)}.first["name"] - assert_nil new_agent["affiliations"].select{|x| x["id"].eql?(agent.affiliations[1].id.to_s)}.first - assert_equal new_affiliations[1][:name], new_agent["affiliations"].reject{|x| x["id"].eql?(agent.affiliations[0].id.to_s)}.first["name"] + assert_equal new_affiliations.size, new_agent['affiliations'].size + assert_equal new_affiliations[0][:name], new_agent['affiliations'].select{|x| x['id'].eql?(agent.affiliations[0].id.to_s)}.first['name'] + assert_nil new_agent['affiliations'].select{|x| x['id'].eql?(agent.affiliations[1].id.to_s)}.first + assert_equal new_affiliations[1][:name], new_agent['affiliations'].reject{|x| x['id'].eql?(agent.affiliations[0].id.to_s)}.first['name'] end def test_delete_agent @@ -189,22 +189,22 @@ def _delete_agents end def _test_agent_creation(agent) - post "/agents", MultiJson.dump(agent), "CONTENT_TYPE" => "application/json" + post '/agents', MultiJson.dump(agent), 'CONTENT_TYPE' => 'application/json' assert last_response.status == 201 created_agent = MultiJson.load(last_response.body) - assert created_agent["name"].eql?(agent[:name]) + assert created_agent['name'].eql?(agent[:name]) - get "/agents/#{created_agent['id'].split('/').last}" + get "/agents/#{created_agent['@id'].split('/').last}" assert last_response.ok? created_agent = MultiJson.load(last_response.body) - assert_equal agent[:name], created_agent["name"] - assert_equal agent[:identifiers].size, created_agent["identifiers"].size - assert_equal agent[:identifiers].map { |x| x[:notation] }.sort, created_agent["identifiers"].map { |x| x['notation'] }.sort + assert_equal agent[:name], created_agent['name'] + assert_equal agent[:identifiers].size, created_agent['identifiers'].size + assert_equal agent[:identifiers].map { |x| x[:notation] }.sort, created_agent['identifiers'].map { |x| x['notation'] }.sort - assert_equal agent[:affiliations].size, created_agent["affiliations"].size - assert_equal agent[:affiliations].map { |x| x["name"] }.sort, created_agent["affiliations"].map { |x| x['name'] }.sort + assert_equal agent[:affiliations].size, created_agent['affiliations'].size + assert_equal agent[:affiliations].map { |x| x['name'] }.sort, created_agent['affiliations'].map { |x| x['name'] }.sort created_agent end end \ No newline at end of file From 4f3508fe42379b7a365e795cd38126c30b37e5ff Mon Sep 17 00:00:00 2001 From: Syphax Date: Fri, 16 May 2025 21:31:04 +0200 Subject: [PATCH 06/23] Revert "Feature: add / route for SemanticartefactCatalog (#117)" This reverts commit f3efd634846ffb20dcd344e2f2bcb2182ce60187. --- Gemfile | 2 +- Gemfile.lock | 4 +- controllers/home_controller.rb | 50 +++---------------- controllers/submission_metadata_controller.rb | 3 -- 4 files changed, 11 insertions(+), 48 deletions(-) diff --git a/Gemfile b/Gemfile index 789aacaef..1b82e07de 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ gem 'json-schema' gem 'multi_json' gem 'oj' gem 'parseconfig' -gem 'rack' +gem 'rack', '3.1.10' gem 'rake' gem 'rexml' # Investigate why unicorn fails to start under ruby 3 without adding rexml gem to the Gemfile gem 'sinatra' diff --git a/Gemfile.lock b/Gemfile.lock index 768e629d6..b19c83d9b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -385,7 +385,7 @@ GEM public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.14) + rack (3.1.10) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -569,7 +569,7 @@ DEPENDENCIES pandoc-ruby parallel parseconfig - rack + rack (= 3.1.10) rack-accept rack-attack rack-cache diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 7f3280795..767ea07f6 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -31,36 +31,15 @@ class HomeController < ApplicationController routes_hash[route_no_slash] = LinkedData.settings.rest_url_prefix + route_no_slash end - catalog_class = LinkedData::Models::SemanticArtefactCatalog - catalog = catalog_class.all.first || create_catalog - attributes_to_include = includes_param[0] == :all ? catalog_class.attributes(:all) : catalog_class.goo_attrs_to_load(includes_param) - catalog.bring(*attributes_to_include) - if catalog.loaded_attributes.include?(:federated_portals) - catalog.federated_portals = catalog.federated_portals.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - catalog.federated_portals.each { |item| item.delete('apikey') } - end - if catalog.loaded_attributes.include?(:fundedBy) - catalog.fundedBy = catalog.fundedBy.map { |item| JSON.parse(item.gsub('=>', ':').gsub('\"', '"')) } - end - catalog.class.link_to *routes_hash.map { |key, url| LinkedData::Hypermedia::Link.new(key, url, context[key]) } - - reply catalog - end + config = LinkedData::Models::PortalConfig.current_portal_config - patch do - catalog = LinkedData::Models::SemanticArtefactCatalog.where.first - error 422, "There is no catalog configs in the triple store" if catalog.nil? - populate_from_params(catalog, params) - if catalog.valid? - catalog.save - status 200 - else - error 422, catalog.errors - end - end + federated_portals = config.federated_portals + federated_portals.transform_values! { |v| v.delete(:apikey); v } + config.init_federated_portals_settings(federated_portals) + config.id = RDF::URI.new(LinkedData.settings.id_url_prefix) + config.class.link_to *routes_hash.map { |key, url| LinkedData::Hypermedia::Link.new(key, url, context[key]) } - get "doc/api" do - redirect "/documentation", 301 + reply config end get "documentation" do @@ -70,20 +49,7 @@ class HomeController < ApplicationController private - def create_catalog - catalog = nil - catalogs = LinkedData::Models::SemanticArtefactCatalog.all - if catalogs.nil? || catalogs.empty? - catalog = instance_from_params(LinkedData::Models::SemanticArtefactCatalog, {"test_attr_to_persist" => "test_to_persist"}) - if catalog.valid? - catalog.save - else - error 422, catalog.errors - end - end - catalog - end - + end end diff --git a/controllers/submission_metadata_controller.rb b/controllers/submission_metadata_controller.rb index 7007f837c..db6fbb78c 100644 --- a/controllers/submission_metadata_controller.rb +++ b/controllers/submission_metadata_controller.rb @@ -13,7 +13,4 @@ class SubmissionMetadataController < ApplicationController reply klass_metadata(LinkedData::Models::Ontology, "ontology_metadata") end - get "/catalog_metadata" do - reply klass_metadata(LinkedData::Models::SemanticArtefactCatalog, "catalog_metadata") - end end \ No newline at end of file From 6f0837a307befaac0f054e4989c0007f286e38fb Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 21 May 2025 20:59:22 +0200 Subject: [PATCH 07/23] Feature: finalize mod API and documentation (#152) * add swagger documentation * add resources and search routes documentation * Finish swagger documentation, change mod namespace and refactor code * fix record test * update Gemfile and Gemfile.lock * update Gemfile.lock to fix tests * address request changes * add search and documentation tests * fix tests * refactore code --- Gemfile.lock | 88 ++++++------ controllers/application_controller.rb | 86 ++++++++++++ controllers/documentation_controller.rb | 33 +++++ controllers/home_controller.rb | 8 +- controllers/mod/artefacts_controller.rb | 92 ++++++++++++ controllers/mod/artefacts_data.rb | 66 --------- controllers/mod/artefacts_metadata.rb | 87 ------------ controllers/mod/mod_search_controller.rb | 33 +++++ controllers/mod/records_controller.rb | 31 ++++ controllers/mod/resources_controller.rb | 90 ++++++++++++ controllers/search_controller.rb | 125 +---------------- .../{artefact_helper.rb => mod_api_helper.rb} | 38 ++++- helpers/openapi_helper.rb | 103 ++++++++++++++ helpers/search_helper.rb | 85 ++++++++++- helpers/swagger_ui_helper.rb | 48 +++++++ ...ntroller.rb => test_mod_api_controller.rb} | 132 ++++++++++++++---- 16 files changed, 786 insertions(+), 359 deletions(-) create mode 100644 controllers/documentation_controller.rb create mode 100644 controllers/mod/artefacts_controller.rb delete mode 100644 controllers/mod/artefacts_data.rb delete mode 100644 controllers/mod/artefacts_metadata.rb create mode 100644 controllers/mod/mod_search_controller.rb create mode 100644 controllers/mod/records_controller.rb create mode 100644 controllers/mod/resources_controller.rb rename helpers/{artefact_helper.rb => mod_api_helper.rb} (69%) create mode 100644 helpers/openapi_helper.rb create mode 100644 helpers/swagger_ui_helper.rb rename test/controllers/{test_artefacts_controller.rb => test_mod_api_controller.rb} (58%) diff --git a/Gemfile.lock b/Gemfile.lock index cc2aec5e9..f98ce3f4a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: e48a2d13a65cc2dd1c12d116cfc9da9061106861 + revision: 04680ed78dfd98cfe004d9a1d7019f3f06e9b667 branch: development specs: goo (0.0.2) @@ -18,7 +18,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git - revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0 + revision: aeb0222400f1b423cb865545c41233d2cbd82bfc branch: development specs: ncbo_annotator (0.0.1) @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: cc4cd9218db7181c4843772631b7f3a96c74a4aa + revision: 3cf75117e8023115cfdec1c8d5a3d78f273d19db branch: master specs: ncbo_cron (0.0.1) @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 913ff00af28225c00133ba41cc2d51eb08cd90ff + revision: 28d4083675acfc852c63198931e8ea5b3edfdfbc branch: development specs: ontologies_linked_data (0.0.1) @@ -169,7 +169,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.3.5) - connection_pool (2.5.2) + connection_pool (2.5.3) crack (0.4.5) rexml dante (0.2.0) @@ -177,8 +177,8 @@ GEM declarative (0.0.20) docile (1.4.1) domain_name (0.6.20240107) - drb (2.2.1) - ed25519 (1.3.0) + drb (2.2.3) + ed25519 (1.4.0) et-orbi (1.2.11) tzinfo faraday (2.13.1) @@ -193,7 +193,7 @@ GEM fugit (1.11.1) et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) - gapic-common (0.25.0) + gapic-common (1.0.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) google-cloud-env (~> 2.2) @@ -205,15 +205,15 @@ GEM grpc (~> 1.66) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.7.0) + google-analytics-data (0.7.1) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.16.0) - gapic-common (>= 0.25.0, < 2.a) + google-analytics-data-v1beta (0.17.0) + gapic-common (~> 1.0) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.16.0) + google-apis-analytics_v3 (0.17.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.16.0) + google-apis-core (0.17.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -224,28 +224,19 @@ GEM google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.2.2) + google-cloud-env (2.3.0) base64 (~> 0.2) faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) - google-logging-utils (0.1.0) - google-protobuf (4.30.2) + google-logging-utils (0.2.0) + google-protobuf (4.31.0) bigdecimal rake (>= 13) - google-protobuf (4.30.2-arm64-darwin) - bigdecimal - rake (>= 13) - google-protobuf (4.30.2-x86_64-darwin) - bigdecimal - rake (>= 13) - google-protobuf (4.30.2-x86_64-linux) - bigdecimal - rake (>= 13) - googleapis-common-protos (1.7.0) + googleapis-common-protos (1.8.0) google-protobuf (>= 3.18, < 5.a) - googleapis-common-protos-types (~> 1.7) + googleapis-common-protos-types (~> 1.20) grpc (~> 1.41) - googleapis-common-protos-types (1.19.0) + googleapis-common-protos-types (1.20.0) google-protobuf (>= 3.18, < 5.a) googleauth (1.14.0) faraday (>= 1.0, < 3.a) @@ -255,22 +246,22 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.71.0) + grpc (1.72.0) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-arm64-darwin) + grpc (1.72.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-x86_64-darwin) + grpc (1.72.0-x86_64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.71.0-x86_64-linux) + grpc (1.72.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) temple (>= 0.8.0) tilt - hashdiff (1.1.2) + hashdiff (1.2.0) htmlentities (4.3.4) http-accept (1.7.0) http-cookie (1.0.8) @@ -279,7 +270,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.11.3) + json (2.12.0) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -294,8 +285,8 @@ GEM jwt (2.10.1) base64 kgio (2.11.4) - language_server-protocol (3.17.0.4) - libxml-ruby (5.0.3) + language_server-protocol (3.17.0.5) + libxml-ruby (5.0.4) link_header (0.0.8) lint_roller (1.1.0) logger (1.7.0) @@ -307,10 +298,10 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.6.2) + mime-types (3.7.0) logger - mime-types-data (~> 3.2015) - mime-types-data (3.2025.0422) + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2025.0520) mini_mime (1.1.5) minitest (5.25.5) minitest-fail-fast (0.1.0) @@ -336,7 +327,7 @@ GEM uri net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.5.7) + net-imap (0.5.8) date net-protocol net-pop (0.1.2) @@ -351,7 +342,7 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.18.0) + newrelic_rpm (9.19.0) oj (3.16.10) bigdecimal (>= 3.0) ostruct (>= 0.2) @@ -371,10 +362,10 @@ GEM pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (6.0.1) + public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.13) + rack (3.1.15) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -383,11 +374,12 @@ GEM rack (>= 0.4) rack-contrib (2.5.0) rack (< 4) - rack-cors (2.0.2) - rack (>= 2.0.0) + rack-cors (3.0.0) + logger + rack (>= 3.0.14) rack-mini-profiler (3.3.1) rack (>= 1.2.0) - rack-session (2.1.0) + rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) @@ -438,7 +430,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.75.3) + rubocop (1.75.6) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -459,7 +451,7 @@ GEM rufus-scheduler (3.9.2) fugit (~> 1.1, >= 1.11.1) securerandom (0.4.1) - signet (0.19.0) + signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) diff --git a/controllers/application_controller.rb b/controllers/application_controller.rb index 5693f8c16..1b16a56d7 100644 --- a/controllers/application_controller.rb +++ b/controllers/application_controller.rb @@ -1,5 +1,8 @@ # This is the base class for controllers in the application. # Code in the before or after blocks will run on every request +require_relative '../helpers/swagger_ui_helper' +require_relative '../helpers/openapi_helper' + class ApplicationController include Sinatra::Delegator extend Sinatra::Delegator @@ -12,4 +15,87 @@ class ApplicationController after { } + register Sinatra::OpenAPIHelper + + configure do + set :app_name, 'MOD-API Documentation' + set :api_version, '1.0.0' + set :api_description, 'Ontoportal MOD-API documentation' + set :base_url, LinkedData.settings.rest_url_prefix + + set :api_schemas, { + hydraPage: { + type: 'object', + required: ['@context', '@id', '@type', 'totalItems', 'itemsPerPage', 'member', 'view'], + properties: { + '@context': { + type: 'object' + }, + '@id': { type: 'string', format: 'uri' }, + '@type': { type: 'string', enum: ['hydra:Collection'] }, + 'totalItems': { type: 'integer' }, + 'itemsPerPage': { type: 'integer' }, + 'view': { + type: 'object', + required: ['@id', '@type'], + properties: { + '@id': { type: 'string', format: 'uri' }, + '@type': { type: 'string', enum: ['hydra:PartialCollectionView'] }, + 'firstPage': { type: 'string', format: 'uri' }, + 'previousPage': { type: 'string', format: 'uri' }, + 'nextPage': { type: 'string', format: 'uri' }, + 'lastPage': { type: 'string', format: 'uri' } + } + }, + 'member': { + type: 'array', + items: { type: 'object' } + } + } + }, + modSemanticArtefact: { + type: 'object', + properties: { + '@id': { type: 'string', format: 'uri'}, + '@type': { type: 'string', const: 'https://w3id.org/mod#modSemanticArtefact' }, + links: { + type: 'object', + properties: { + link: { type: 'string', format: 'uri' }, + '@context': { type: 'array', items: {type: 'string'} } + } + }, + '@context': { + type: 'object', + properties: { + property: { type: 'string', format: 'uri' }, + } + } + } + }, + + modSemanticArtefactDistribution: { + type: 'object', + properties: { + '@id': { type: 'string', format: 'uri'}, + '@type': { type: 'string', const: 'https://w3id.org/mod#SemanticArtefactDistribution' }, + links: { + type: 'object', + properties: { + link: { type: 'string', format: 'uri' }, + '@context': { type: 'array', items: {type: 'string'} } + } + }, + '@context': { + type: 'object', + properties: { + property: { type: 'string', format: 'uri' }, + } + } + } + } + } + + end + end diff --git a/controllers/documentation_controller.rb b/controllers/documentation_controller.rb new file mode 100644 index 000000000..90d239b49 --- /dev/null +++ b/controllers/documentation_controller.rb @@ -0,0 +1,33 @@ +class DocumentationController < ApplicationController + get '/mod-api/doc/api' do + content_type 'text/html' + <<-HTML + + + + + MOD-API Documentation + + + +
+ + + + + HTML + end + + # Serve OpenAPI JSON + get '/openapi.json' do + content_type :json + generate_openapi_json.to_json + end +end diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 0f485c86d..ebd32e38c 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -4,6 +4,10 @@ class HomeController < ApplicationController namespace '/' do + doc('Catalog', 'Get the semantic artefact catalogue') do + default_params(display: true) + default_responses(success: true) + end get do expires 3600, :public last_modified @@root_last_modified ||= Time.now.httpdate @@ -30,10 +34,6 @@ class HomeController < ApplicationController end end - get "doc/api" do - redirect "/documentation", 301 - end - get "documentation" do @metadata_all = get_metadata_all.sort { |a, b| a[0].name <=> b[0].name } haml "documentation/documentation".to_sym, :layout => "documentation/layout".to_sym diff --git a/controllers/mod/artefacts_controller.rb b/controllers/mod/artefacts_controller.rb new file mode 100644 index 000000000..b29cb12b5 --- /dev/null +++ b/controllers/mod/artefacts_controller.rb @@ -0,0 +1,92 @@ +class ArtefactsController < ApplicationController + namespace "/mod-api" do + namespace "/artefacts" do + + doc('Artefact', 'Get information about all semantic artefacts') do + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefact) + attributes, page, pagesize = settings_params(LinkedData::Models::SemanticArtefact).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefact.goo_attrs_to_load([]) if includes_param.first == :all + artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) + reply artefacts + end + + doc('Artefact', 'Get information about a semantic artefact') do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID" do + artefact = find_artefact(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? + check_last_modified(artefact) + artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) + reply artefact + end + + doc('Artefact', "Get information about a semantic artefact's latest distribution") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true) + end + get "/:artefactID/distributions/latest" do + artefact = find_artefact(params["artefactID"]) + include_status = params["include_status"]&.to_sym || :any + latest_distribution = artefact.latest_distribution(status: include_status) + + if latest_distribution + check_last_modified(latest_distribution) + latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + end + reply latest_distribution + end + + doc('Artefact', "Get information about a semantic artefact's distribution") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('distributionID', type: 'number', description: 'The id of the distribution', default: 5) + default_params(display: true) + default_responses(success: true, not_found: true) + end + get '/:artefactID/distributions/:distributionID' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + artefact_distribution = artefact.distribution(params["distributionID"]) + error 404, "Distribution with ID #{params['distributionID']} not found" if artefact_distribution.nil? + artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) + reply artefact_distribution + end + + doc('Artefact', "Get information about a semantic artefact's distributions") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + default_responses(not_found: true) + end + get '/:artefactID/distributions' do + artefact = find_artefact(params["artefactID"]) + check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + attributes = LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) if includes_param.first == :all + distros = artefact.all_distributions(attributes, page, pagesize) + reply distros + end + + doc('Record', "Get information about a semantic artefact catalog record") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID/record" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + end + end +end diff --git a/controllers/mod/artefacts_data.rb b/controllers/mod/artefacts_data.rb deleted file mode 100644 index 8fefce2d5..000000000 --- a/controllers/mod/artefacts_data.rb +++ /dev/null @@ -1,66 +0,0 @@ -class ArtefactsdataController < ApplicationController - namespace "/artefacts/:artefactID/resources" do - - get do - ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) - check_access(ontology) - _, page, size = settings_params(LinkedData::Models::Class).first(3) - size_per_type = [size / 6, 1].max - - types = [ - LinkedData::Models::Class, - LinkedData::Models::Instance, - LinkedData::Models::SKOS::Scheme, - LinkedData::Models::SKOS::Collection, - LinkedData::Models::SKOS::Label - ] - - total_count = 0 - resources = types.flat_map do |model| - resource_page = load_resources_hydra_page(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_type) - total_count += resource_page.aggregate - resource_page.to_a - end - - props_page = load_properties_hydra_page(ontology, latest_submission, page, size_per_type) - resources.concat(props_page.to_a) - total_count += props_page.aggregate - reply hydra_page_object(resources, total_count) - end - - def self.define_resource_routes(resource_types, expected_type) - resource_types.each do |type| - - get "/#{type}" do - ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) - check_access(ontology) - model_class = (type == 'properties') ? LinkedData::Models::OntologyProperty : model_from_type(type) - attributes, page, size = settings_params(model_class).first(3) - - if type == 'properties' - reply load_properties_hydra_page(ontology, latest_submission, page, size) - else - rdf_type = LinkedData::Models::Class.class_rdf_type(latest_submission) - if rdf_type == expected_type - reply load_resources_hydra_page(ontology, latest_submission, model_class, attributes, page, size) - else - reply hydra_empty_page - end - end - end - - get "/#{type}/:uri" do - reply resolve_resource_by_uri - end - end - end - - define_resource_routes(%w[classes individuals], RDF::OWL[:Class]) - define_resource_routes(%w[concepts schemes collections labels], RDF::Vocab::SKOS[:Concept]) - define_resource_routes(%w[properties], 'properties') - - get '/:uri' do - reply resolve_resource_by_uri - end - end -end diff --git a/controllers/mod/artefacts_metadata.rb b/controllers/mod/artefacts_metadata.rb deleted file mode 100644 index b286f8297..000000000 --- a/controllers/mod/artefacts_metadata.rb +++ /dev/null @@ -1,87 +0,0 @@ -class ArtefactsMetadataController < ApplicationController - namespace "/artefacts" do - # Get all Semantic Artefacts - get do - check_last_modified_collection(LinkedData::Models::SemanticArtefact) - attributes, page, pagesize = settings_params(LinkedData::Models::SemanticArtefact).first(3) - pagesize ||= 20 - attributes = LinkedData::Models::SemanticArtefact.goo_attrs_to_load([]) if includes_param.first == :all - artefacts = LinkedData::Models::SemanticArtefact.all_artefacts(attributes, page, pagesize) - reply artefacts - end - - # Get one semantic artefact by ID - get "/:artefactID" do - artefact = find_artefact(params["artefactID"]) - error 404, "You must provide a valid `artefactID` to retrieve an artefact" if artefact.nil? - check_last_modified(artefact) - artefact.bring(*LinkedData::Models::SemanticArtefact.goo_attrs_to_load(includes_param)) - reply artefact - end - - # Get artefact catalog record by ID - get "/:artefactID/record" do - record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) - error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? - check_last_modified(record) - record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) - reply record - end - - # Display latest distribution of an artefact - get "/:artefactID/distributions/latest" do - artefact = find_artefact(params["artefactID"]) - include_status = params["include_status"]&.to_sym || :any - latest_distribution = artefact.latest_distribution(status: include_status) - - if latest_distribution - check_last_modified(latest_distribution) - latest_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - end - reply latest_distribution - end - - # Display a distribution by ID - get '/:artefactID/distributions/:distributionID' do - artefact = find_artefact(params["artefactID"]) - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - artefact_distribution = artefact.distribution(params["distributionID"]) - error 404, "Distribution with ID #{params['distributionID']} not found" if artefact_distribution.nil? - artefact_distribution.bring(*LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load(includes_param)) - reply artefact_distribution - end - - # Display all distributions of an artefact - get '/:artefactID/distributions' do - artefact = find_artefact(params["artefactID"]) - check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) - attributes = LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) if includes_param.first == :all - distros = artefact.all_distributions(attributes, page, pagesize) - reply distros - end - - end - - namespace "/records" do - # Get all Semantic Artefact Catalog Records - get do - check_last_modified_collection(LinkedData::Models::SemanticArtefactCatalogRecord) - attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) - pagesize ||= 20 - attributes = LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load([]) if includes_param.first == :all - records = LinkedData::Models::SemanticArtefactCatalogRecord.all(attributes, page, pagesize) - reply records - end - - # Get a specific record by artefact ID - get "/:artefactID" do - record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) - error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? - check_last_modified(record) - record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) - reply record - end - - end -end diff --git a/controllers/mod/mod_search_controller.rb b/controllers/mod/mod_search_controller.rb new file mode 100644 index 000000000..9b045ab7c --- /dev/null +++ b/controllers/mod/mod_search_controller.rb @@ -0,0 +1,33 @@ +class ModSearchController < ApplicationController + namespace "/mod-api" do + namespace "/search" do + + doc('Search', 'Search content/metadata of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + result = process_search + reply hydra_page_object(result.to_a, result.aggregate) + end + + doc('Search', 'Search content of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get '/content' do + result = process_search + reply hydra_page_object(result.to_a, result.aggregate) + end + + doc('Search', 'Search metadata of artefacts') do + default_params(display: true, pagination: true, query: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get '/metadata' do + hydra_page_result = search_metadata + reply hydra_page_result + end + end + end +end \ No newline at end of file diff --git a/controllers/mod/records_controller.rb b/controllers/mod/records_controller.rb new file mode 100644 index 000000000..c8c3df208 --- /dev/null +++ b/controllers/mod/records_controller.rb @@ -0,0 +1,31 @@ +class RecordsController < ApplicationController + namespace "/mod-api" do + namespace "/records" do + doc('Record', "Get information about all semantic artefact catalog records") do + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + check_last_modified_collection(LinkedData::Models::SemanticArtefactCatalogRecord) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + pagesize ||= 20 + attributes = LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load([]) if includes_param.first == :all + records = LinkedData::Models::SemanticArtefactCatalogRecord.all(attributes, page, pagesize) + reply records + end + + doc('Record', "Get information about a semantic artefact catalog record") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true) + default_responses(success: true, not_found: true) + end + get "/:artefactID" do + record = LinkedData::Models::SemanticArtefactCatalogRecord.find(params["artefactID"]) + error 404, "You must provide a valid `artefactID` to retrieve ats record" if record.nil? + check_last_modified(record) + record.bring(*LinkedData::Models::SemanticArtefactCatalogRecord.goo_attrs_to_load(includes_param)) + reply record + end + end + end +end diff --git a/controllers/mod/resources_controller.rb b/controllers/mod/resources_controller.rb new file mode 100644 index 000000000..72cc13002 --- /dev/null +++ b/controllers/mod/resources_controller.rb @@ -0,0 +1,90 @@ +class ResourcesController < ApplicationController + namespace "/mod-api" do + namespace "/artefacts/:artefactID/resources" do + + doc('Artefact', "Get a list of all the resources within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + _, page, size = settings_params(LinkedData::Models::Class).first(3) + size_per_type = [size / 6, 1].max + + types = [ + LinkedData::Models::Class, + LinkedData::Models::Instance, + LinkedData::Models::SKOS::Scheme, + LinkedData::Models::SKOS::Collection, + LinkedData::Models::SKOS::Label + ] + + total_count = 0 + resources = types.flat_map do |model| + resource_page = load_resources_hydra_page(ontology, latest_submission, model, model.goo_attrs_to_load([]), page, size_per_type) + total_count += resource_page.aggregate + resource_page.to_a + end + + props_page = load_properties_hydra_page(ontology, latest_submission, page, size_per_type) + resources.concat(props_page.to_a) + total_count += props_page.aggregate + reply hydra_page_object(resources, total_count) + end + + def self.define_resource_routes(resource_types, expected_type) + resource_types.each do |type| + + doc('Artefact', "Get a list of all #{type} within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + default_params(display: true, pagination: true) + response(200, "OK", content('$ref' => '#/components/schemas/hydraPage')) + end + get "/#{type}" do + ontology, latest_submission = get_ontology_and_submission(ontology_acronym: params["artefactID"]) + check_access(ontology) + model_class = (type == 'properties') ? LinkedData::Models::OntologyProperty : model_from_type(type) + attributes, page, size = settings_params(model_class).first(3) + + if type == 'properties' + reply load_properties_hydra_page(ontology, latest_submission, page, size) + else + rdf_type = LinkedData::Models::Class.class_rdf_type(latest_submission) + if rdf_type == expected_type + reply load_resources_hydra_page(ontology, latest_submission, model_class, attributes, page, size) + else + reply hydra_empty_page + end + end + end + + + doc('Artefact', "Get specific #{type} of a semantic artefact by it's uri") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('uri', type: 'string', description: 'The uri of the resource', default: "FAKE_URI") + default_responses(success: true, not_found: true) + end + get "/#{type}/:uri" do + reply resolve_resource_by_uri + end + end + end + + define_resource_routes(%w[classes individuals], RDF::OWL[:Class]) + define_resource_routes(%w[concepts schemes collections labels], RDF::Vocab::SKOS[:Concept]) + define_resource_routes(%w[properties], 'properties') + + + doc('Artefact', "Get a specific resources from within an artefact") do + path_parameter('artefactID', type: 'string', description: 'The acronym of the artefact', default: "STY") + path_parameter('uri', type: 'string', description: 'The uri of the resource', default: "FAKE_URI") + default_responses(success: true, not_found: true) + end + get '/:uri' do + reply resolve_resource_by_uri + end + end + end +end diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 82aa394d0..5e340f9a6 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -4,50 +4,15 @@ class SearchController < ApplicationController namespace "/search" do # execute a search query - get do - process_search - end - - get '/content' do - process_search - end - get '/metadata' do - query = get_query(params) - options = get_ontology_metadata_search_options(params) - page, page_size = page_params - - resp = search(Ontology, query, options) - - result = {} - acronyms_ids = {} - resp.each do |doc| - id = doc["submissionId_i"] - acronym = doc["ontology_acronym_text"] || doc["ontology_t"]&.split('/')&.last - next if acronym.blank? - - old_id = acronyms_ids[acronym].to_i rescue 0 - already_found = (old_id && id && (id <= old_id)) - - next if already_found - - not_restricted = (doc["ontology_viewingRestriction_t"]&.eql?('public') || current_user&.admin?) - user_not_restricted = not_restricted || - Array(doc["ontology_viewingRestriction_txt"]).any? {|u| u.split(' ').last == current_user&.username} || - Array(doc["ontology_acl_txt"]).any? {|u| u.split(' ').last == current_user&.username} - - user_restricted = !user_not_restricted - next if user_restricted - - acronyms_ids[acronym] = id - result[acronym] = LinkedData::Models::SemanticArtefact.read_only(id: "#{LinkedData.settings.id_url_prefix}artefacts/#{acronym}", acronym: acronym, description: doc['description_text'], title: doc['ontology_name_text']) - end - - reply hydra_page_object(result.values, result.length) + get do + page = process_search + reply 200, page end post do - process_search + page = process_search + reply 200, page end namespace "/ontologies" do @@ -171,85 +136,5 @@ class SearchController < ApplicationController end end - private - - def search(model, query, params = {}) - query = query.blank? ? "*" : query - - resp = model.search(query, search_params(**params)) - - total_found = resp["response"]["numFound"] - docs = resp["response"]["docs"] - - page_object(docs, total_found) - end - - def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOperators: "true", page:, page_size:, fl: '*,score', sort:) - { - defType: defType, - fq: fq, - qf: qf, - sort: sort, - start: (page - 1) * page_size, - rows: page_size, - fl: fl, - stopwords: stopwords, - lowercaseOperators: lowercaseOperators, - } - end - - def process_search(params = nil) - params ||= @params - params['q'] ||= params['query'] - params.delete('query') - text = params["q"] - - query = get_term_search_query(text, params) - # puts "Edismax query: #{query}, params: #{params}" - set_page_params(params) - - docs = Array.new - resp = LinkedData::Models::Class.search(query, params) - total_found = resp["response"]["numFound"] - add_matched_fields(resp, Sinatra::Helpers::SearchHelper::MATCH_TYPE_PREFLABEL) - ontology_rank = LinkedData::Models::Ontology.rank - - resp["response"]["docs"].each do |doc| - doc = doc.symbolize_keys - # NCBO-974 - doc[:matchType] = resp["match_types"][doc[:id]] - resource_id = doc[:resource_id] - doc.delete :resource_id - doc[:id] = resource_id - # TODO: The `rescue next` on the following line shouldn't be here - # However, at some point we didn't store the ontologyId in the index - # and these records haven't been cleared out so this is getting skipped - ontology_uri = doc[:ontologyId].sub(/\/submissions\/.*/, "") rescue next - ontology = LinkedData::Models::Ontology.read_only(id: ontology_uri, acronym: doc[:submissionAcronym]) - submission = LinkedData::Models::OntologySubmission.read_only(id: doc[:ontologyId], ontology: ontology) - doc[:submission] = submission - doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 - doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) - - doc = filter_attrs_by_language(doc) - - instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) - docs.push(instance) - end - - unless params['sort'] - if !text.nil? && text[-1] == '*' - docs.sort! { |a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]] } - else - docs.sort! { |a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]] } - end - end - - # need to return a Page object - page = page_object(docs, total_found) - - reply 200, page - end - end end diff --git a/helpers/artefact_helper.rb b/helpers/mod_api_helper.rb similarity index 69% rename from helpers/artefact_helper.rb rename to helpers/mod_api_helper.rb index 5df0ee5eb..479837f81 100644 --- a/helpers/artefact_helper.rb +++ b/helpers/mod_api_helper.rb @@ -2,7 +2,7 @@ module Sinatra module Helpers - module ArtefactHelper + module ModApiHelper def load_resources_hydra_page(ont, latest_submission, model, attributes, page, size) check_last_modified_segment(model, [@params["artefactID"]]) @@ -78,8 +78,42 @@ def find_artefact(artefact_id) artefact end + def search_metadata + query = get_query(params) + options = get_ontology_metadata_search_options(params) + page, page_size = page_params + + resp = search(Ontology, query, options) + + result = {} + acronyms_ids = {} + resp.each do |doc| + id = doc["submissionId_i"] + acronym = doc["ontology_acronym_text"] || doc["ontology_t"]&.split('/')&.last + next if acronym.blank? + + old_id = acronyms_ids[acronym].to_i rescue 0 + already_found = (old_id && id && (id <= old_id)) + + next if already_found + + not_restricted = (doc["ontology_viewingRestriction_t"]&.eql?('public') || current_user&.admin?) + user_not_restricted = not_restricted || + Array(doc["ontology_viewingRestriction_txt"]).any? {|u| u.split(' ').last == current_user&.username} || + Array(doc["ontology_acl_txt"]).any? {|u| u.split(' ').last == current_user&.username} + + user_restricted = !user_not_restricted + next if user_restricted + + acronyms_ids[acronym] = id + result[acronym] = LinkedData::Models::SemanticArtefact.read_only(id: "#{LinkedData.settings.id_url_prefix}artefacts/#{acronym}", acronym: acronym, description: doc['description_text'], title: doc['ontology_name_text']) + end + + return hydra_page_object(result.values, result.length) + end + end end end -helpers Sinatra::Helpers::ArtefactHelper +helpers Sinatra::Helpers::ModApiHelper diff --git a/helpers/openapi_helper.rb b/helpers/openapi_helper.rb new file mode 100644 index 000000000..3146f8cb8 --- /dev/null +++ b/helpers/openapi_helper.rb @@ -0,0 +1,103 @@ +require 'sinatra/base' +require 'ostruct' + +module Sinatra + module OpenAPIHelper + class OpenAPIDoc + include Sinatra::OpenAPIHelper + Parameter = Struct.new(:name, :in, :required, :type, :description, :default, :schema, keyword_init: true) + Response = Struct.new(:description, :content, keyword_init: true) + + def initialize(tags, summary) + @tags = tags + @summary = summary + @parameters = [] + @responses = {} + end + + def to_hash + { + tags: @tags, + summary: @summary, + parameters: @parameters, + responses: @responses + } + end + + def content(schema, content_type = 'application/json-ld') + { content_type => { schema: schema } } + end + + def response(status, description = nil, content = nil) + @responses[status] = Response.new(description: description, content: content) + end + + def parameter(name, in_: 'query', required: false, type: 'string', description: nil, default: nil, schema: nil) + @parameters << Parameter.new(name: name, in: in_, required: required, type: type, description: description, default: default, schema: schema) + end + + def path_parameter(name, required: true, type: 'string', description: nil, default: nil, schema: nil) + parameter(name, in_: 'path', required: required, type: type, description: description, default: default, schema: schema) + end + + def body_parameter(name, required: true, type: 'object', description: nil, schema: nil) + parameter(name, in_: 'body', required: required, type: type, description: description, schema: schema) + end + end + + def doc(tags = ["default"], summary, &block) + array_tags = tags.is_a?(Array) ? tags : [tags] + doc = OpenAPIDoc.new(array_tags, summary) + doc.instance_eval(&block) + @pending_api_doc = doc.to_hash + end + + def default_params(display: false, pagination: false, query: false) + display_param if display + pagination_params if pagination + query_param if query + end + + def default_responses(success: false, created: false, no_content: false, bad_request: false, unauthorized: false, not_found: false, server_error: false) + response(200, "OK") if success + response(201, "Created") if created + response(204, "No Content") if no_content + response(400, "Bad Request") if bad_request + response(401, "Unauthorized") if unauthorized + response(404, "Not Found") if not_found + response(500, "Internal Server Error") if server_error + end + + def display_param + parameter('display', type: 'string', description: 'Attributes to display', default: '') + end + + def pagination_params + parameter('page', type: 'integer', description: 'Page number', default: '1') + parameter('pagesize', type: 'integer', description: 'Number of items per page', default: '20') + end + + def query_param + parameter('q', type: 'string', description: 'Query text', default: 'plant') + end + + def self.registered(app) + app.before do + @pending_api_doc = nil + end + end + + def route(verb, path, opts = {}, &block) + if @pending_api_doc + @api_docs ||= {} + @api_docs[path.first] ||= {} + @api_docs[path.first][verb.downcase] = @pending_api_doc + @pending_api_doc = nil + end + super(verb, path, opts, &block) + end + end +end + + + diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index d88a42f8c..1e0d2d030 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -448,8 +448,8 @@ def validate_params_solr_population(allowed_includes_params) message = "The `include` query string parameter cannot accept #{leftover.join(", ")}, please use only #{allowed_includes_params.join(", ")}" error 400, message if invalid end - - + + def get_ontology_metadata_search_options(params) groups = params.fetch("groups", "").split(',') categories = params.fetch("hasDomain", "").split(',') @@ -491,7 +491,7 @@ def get_ontology_metadata_search_options(params) "ontologySuggestNgram^2 ontology_acronymSuggestNgram^2 ontology_nameSuggestNgram^1.5 descriptionSuggestNgram" # substring match last ].join(' ') end - + options = { fq: fq, qf: qf, @@ -501,7 +501,7 @@ def get_ontology_metadata_search_options(params) } options end - + def get_query(params) if params[:query].nil? && params[:q].nil? raise error 400, "The search query must be provided via /search?q=[&page=&pagesize=] /search?query=[&page=&pagesize=]" @@ -509,7 +509,82 @@ def get_query(params) query = params[:query] || params[:q] query end - + + def search(model, query, params = {}) + query = query.blank? ? "*" : query + + resp = model.search(query, search_params(**params)) + + total_found = resp["response"]["numFound"] + docs = resp["response"]["docs"] + + page_object(docs, total_found) + end + + def search_params(defType: "edismax", fq:, qf:, stopwords: "true", lowercaseOperators: "true", page:, page_size:, fl: '*,score', sort:) + { + defType: defType, + fq: fq, + qf: qf, + sort: sort, + start: (page - 1) * page_size, + rows: page_size, + fl: fl, + stopwords: stopwords, + lowercaseOperators: lowercaseOperators, + } + end + + def process_search(params = nil) + params ||= @params + params['q'] ||= params['query'] + params.delete('query') + text = params["q"] + + query = get_term_search_query(text, params) + # puts "Edismax query: #{query}, params: #{params}" + set_page_params(params) + + docs = Array.new + resp = LinkedData::Models::Class.search(query, params) + total_found = resp["response"]["numFound"] + add_matched_fields(resp, Sinatra::Helpers::SearchHelper::MATCH_TYPE_PREFLABEL) + ontology_rank = LinkedData::Models::Ontology.rank + + resp["response"]["docs"].each do |doc| + doc = doc.symbolize_keys + # NCBO-974 + doc[:matchType] = resp["match_types"][doc[:id]] + resource_id = doc[:resource_id] + doc.delete :resource_id + doc[:id] = resource_id + # TODO: The `rescue next` on the following line shouldn't be here + # However, at some point we didn't store the ontologyId in the index + # and these records haven't been cleared out so this is getting skipped + ontology_uri = doc[:ontologyId].sub(/\/submissions\/.*/, "") rescue next + ontology = LinkedData::Models::Ontology.read_only(id: ontology_uri, acronym: doc[:submissionAcronym]) + submission = LinkedData::Models::OntologySubmission.read_only(id: doc[:ontologyId], ontology: ontology) + doc[:submission] = submission + doc[:ontology_rank] = (ontology_rank[doc[:submissionAcronym]] && !ontology_rank[doc[:submissionAcronym]].empty?) ? ontology_rank[doc[:submissionAcronym]][:normalizedScore] : 0.0 + doc[:properties] = MultiJson.load(doc.delete(:propertyRaw)) if include_param_contains?(:properties) + + doc = filter_attrs_by_language(doc) + + instance = doc[:provisional] ? LinkedData::Models::ProvisionalClass.read_only(doc) : LinkedData::Models::Class.read_only(doc) + docs.push(instance) + end + + unless params['sort'] + if !text.nil? && text[-1] == '*' + docs.sort! { |a, b| [b[:score], a[:prefLabelExact].downcase, b[:ontology_rank]] <=> [a[:score], b[:prefLabelExact].downcase, a[:ontology_rank]] } + else + docs.sort! { |a, b| [b[:score], b[:ontology_rank]] <=> [a[:score], a[:ontology_rank]] } + end + end + + page_object(docs, total_found) + end + end end end diff --git a/helpers/swagger_ui_helper.rb b/helpers/swagger_ui_helper.rb new file mode 100644 index 000000000..5428a9fb7 --- /dev/null +++ b/helpers/swagger_ui_helper.rb @@ -0,0 +1,48 @@ +require 'json' +require 'sinatra/base' + + +module Sinatra + module SwaggerUI + def generate_openapi_json + { + openapi: '3.0.0', + info: { + title: settings.app_name || 'MOD-API Documentation', + version: settings.api_version || '1.0.0', + description: settings.api_description || 'MOD-API Documentation' + }, + servers: [ + { + url: settings.base_url || '/' + } + ], + tags: [ + { name: 'Artefact', description: 'Get information about semantic artefact(s) (ontologies, terminologies, taxonomies, thesauri, vocabularies, metadata schemas and semantic standards) or their resources.' }, + { name: 'Catalog', description: 'Get information about the semantic artefact catalogue.' }, + { name: 'Record', description: 'Get semantic artefact catalogue records' }, + { name: 'Search', description: 'Search the metadata and catalogue content.' } + ], + paths: generate_paths, + components: { + schemas: settings.respond_to?(:api_schemas) ? settings.api_schemas : {} + } + } + end + + def generate_paths + paths = {} + api_docs = settings.instance_variable_get(:@api_docs) + sorted_paths = api_docs.keys.sort_by do |path| + path.is_a?(Mustermann::Sinatra) ? path.to_s : path + end + + sorted_paths.each do |path| + paths[path] = api_docs[path].transform_keys(&:to_s) + end + paths + end + end +end + +helpers Sinatra::SwaggerUI diff --git a/test/controllers/test_artefacts_controller.rb b/test/controllers/test_mod_api_controller.rb similarity index 58% rename from test/controllers/test_artefacts_controller.rb rename to test/controllers/test_mod_api_controller.rb index 7269c2c7f..2ae052322 100644 --- a/test/controllers/test_artefacts_controller.rb +++ b/test/controllers/test_mod_api_controller.rb @@ -18,7 +18,6 @@ def self._create_onts submissions_to_process: [1], process_submission: true, random_submission_count: false, - process_options: {process_rdf: true, extract_metadata: false}, acronym: "TST" } # this will create 2 ontologies (TST-0, TST-1) with 2 submissions each @@ -28,6 +27,7 @@ def self._create_onts @@ontology_type = type == RDF::OWL[:Class] ? "OWL" : "SKOS" @@page = 2 @@pagesize = 1 + @@ontologies[0].latest_submission.index_all(Logger.new($stdout)) end def test_home_controller @@ -80,18 +80,19 @@ def test_home_controller def test_all_artefacts - route = '/artefacts' + route = '/mod-api/artefacts' get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? artefacts_page_data = MultiJson.load(last_response.body) - validate_hydra_page(route, artefacts_page_data, @@num_onts_created) + validate_hydra_page(route, artefacts_page_data) + assert_equal @@num_onts_created, artefacts_page_data["totalItems"] artefacts_page_data["member"].each do |artefact| assert @@created_ont_acronyms.include?(artefact["acronym"]) end end def test_one_artefact - route = "/artefacts/#{@@ontology_0_acronym}" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}" get route assert last_response.ok? artefact_data = MultiJson.load(last_response.body) @@ -99,15 +100,16 @@ def test_one_artefact end def test_all_distributions - route = "/artefacts/#{@@ontology_0_acronym}/distributions" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? dists_page_data = MultiJson.load(last_response.body) - validate_hydra_page(route, dists_page_data, 2) + validate_hydra_page(route, dists_page_data) + assert_equal 2, dists_page_data["totalItems"] end def test_one_distribution - route = "/artefacts/#{@@ontology_0_acronym}/distributions/1" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions/1" get route assert last_response.ok? dist_data = MultiJson.load(last_response.body) @@ -115,7 +117,7 @@ def test_one_distribution end def test_latest_distribution - route = "/artefacts/#{@@ontology_0_acronym}/distributions/latest" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/distributions/latest" get route assert last_response.ok? dist_data = MultiJson.load(last_response.body) @@ -124,70 +126,84 @@ def test_latest_distribution def test_resources total_count = total_resources_count - route = "/artefacts/#{@@ontology_0_acronym}/resources" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? resources_page_data = MultiJson.load(last_response.body) - validate_hydra_page(route, resources_page_data, total_count) + validate_hydra_page(route, resources_page_data) + assert_equal total_count, resources_page_data["totalItems"] + end + + def test_one_resource + uri = "http://bioontology.org/ontologies/BiomedicalResourceOntology.owl#Modular_Component" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{CGI.escape(uri)}" + get route + assert last_response.ok? + resource_data = MultiJson.load(last_response.body) + assert_equal uri, resource_data["@id"] end %w[classes individuals].each do |resource| define_method("test_#{resource}") do - route = "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? page_data = MultiJson.load(last_response.body) if @@ontology_type == "OWL" resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) - validate_hydra_page(route, page_data, resource_count) + validate_hydra_page(route, page_data) + assert_equal resource_count, page_data["totalItems"] else - validate_hydra_page(route, page_data, 0) + validate_hydra_page(route, page_data) end end end %w[concepts schemes collections labels].each do |resource| define_method("test_#{resource}") do - route = "/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/#{resource}" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? page_data = MultiJson.load(last_response.body) if @@ontology_type == "SKOS" resource_count = model_count(resource_model[resource], @@ontology_0.latest_submission) - validate_hydra_page(route, page_data, resource_count) + validate_hydra_page(route, page_data) + assert_equal resource_count, page_data["totalItems"] else - validate_hydra_page(route, page_data, 0) + validate_hydra_page(route, page_data) end end end def test_properties - route = "/artefacts/#{@@ontology_0_acronym}/resources/properties" + route = "/mod-api/artefacts/#{@@ontology_0_acronym}/resources/properties" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? properties_page_data = MultiJson.load(last_response.body) properties_count = @@ontology_0.properties.count - validate_hydra_page(route, properties_page_data, properties_count) + validate_hydra_page(route, properties_page_data) + assert_equal properties_count, properties_page_data["totalItems"] end def test_records - route = "/records" + route = "/mod-api/records" get "#{route}?page=#{@@page}&pagesize=#{@@pagesize}" assert last_response.ok? records_page_data = MultiJson.load(last_response.body) - validate_hydra_page(route, records_page_data, @@num_onts_created) + validate_hydra_page(route, records_page_data) + assert_equal @@num_onts_created, records_page_data["totalItems"] records_page_data["member"].each do |artefact| assert @@created_ont_acronyms.include?(artefact["acronym"]) end end def test_one_record - get "/records/#{@@ontology_0_acronym}" + get "/mod-api/records/#{@@ontology_0_acronym}" assert last_response.ok? record_data_from_records = MultiJson.load(last_response.body) assert_equal @@ontology_0_acronym, record_data_from_records["acronym"] - get "/artefacts/#{@@ontology_0_acronym}/record" + get "/mod-api/artefacts/#{@@ontology_0_acronym}/record" assert last_response.ok? record_data_from_artefact = MultiJson.load(last_response.body) assert_equal @@ontology_0_acronym, record_data_from_artefact["acronym"] @@ -195,20 +211,82 @@ def test_one_record assert_equal record_data_from_artefact, record_data_from_records end + def test_search_content + route = "/mod-api/search/content" + get "#{route}?query=modular" + assert last_response.ok? + search_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, search_page_data) + end + + def test_search_metadata + route = "/mod-api/search/metadata" + get "#{route}?query=TST-0" + assert last_response.ok? + search_page_data = MultiJson.load(last_response.body) + validate_hydra_page(route, search_page_data) + end + + def test_swagger_documentation + get "/openapi.json" + assert last_response.ok? + assert_equal 'application/json', last_response.content_type + + doc = JSON.parse(last_response.body) + + assert_equal '3.0.0', doc['openapi'] + assert_equal 'MOD-API Documentation', doc['info']['title'] + assert_equal '1.0.0', doc['info']['version'] + assert_equal 'Ontoportal MOD-API documentation', doc['info']['description'] + + expected_paths = [ + '/', + '/mod-api/artefacts', + '/mod-api/artefacts/{artefactID}', + '/mod-api/artefacts/{artefactID}/distributions', + '/mod-api/artefacts/{artefactID}/distributions/latest', + '/mod-api/artefacts/{artefactID}/distributions/{distributionID}', + '/mod-api/artefacts/{artefactID}/record', + '/mod-api/artefacts/{artefactID}/resources', + '/mod-api/artefacts/{artefactID}/resources/classes', + '/mod-api/artefacts/{artefactID}/resources/classes/{uri}', + '/mod-api/artefacts/{artefactID}/resources/collections', + '/mod-api/artefacts/{artefactID}/resources/collections/{uri}', + '/mod-api/artefacts/{artefactID}/resources/concepts', + '/mod-api/artefacts/{artefactID}/resources/concepts/{uri}', + '/mod-api/artefacts/{artefactID}/resources/individuals', + '/mod-api/artefacts/{artefactID}/resources/individuals/{uri}', + '/mod-api/artefacts/{artefactID}/resources/labels', + '/mod-api/artefacts/{artefactID}/resources/labels/{uri}', + '/mod-api/artefacts/{artefactID}/resources/properties', + '/mod-api/artefacts/{artefactID}/resources/properties/{uri}', + '/mod-api/artefacts/{artefactID}/resources/schemes', + '/mod-api/artefacts/{artefactID}/resources/schemes/{uri}', + '/mod-api/artefacts/{artefactID}/resources/{uri}', + '/mod-api/records', + '/mod-api/records/{artefactID}', + '/mod-api/search', + '/mod-api/search/content', + '/mod-api/search/metadata' + ] + assert_equal expected_paths.sort, doc['paths'].keys.sort + end + private - def validate_hydra_page(route, page_data, resource_count) + def validate_hydra_page(route, page_data) assert page_data.key?('@context') - assert_equal "#{LinkedData.settings.rest_url_prefix.chomp("/")}#{route}", page_data['@id'] - assert_equal 'hydra:Collection', page_data['@type'] - assert_equal resource_count, page_data["totalItems"] + assert page_data.key?('@id') + assert page_data.key?('@type') + assert page_data.key?("totalItems") assert page_data.key?('itemsPerPage') assert page_data.key?('view') - assert_equal "#{LinkedData.settings.rest_url_prefix.chomp("/")}#{route}?page=#{@@page}&pagesize=#{@@pagesize}", page_data['view']['@id'] + assert page_data['view'].key?('@id') assert page_data['view'].key?('firstPage') assert page_data['view'].key?('previousPage') assert page_data['view'].key?('nextPage') assert page_data['view'].key?('lastPage') + assert page_data.key?('member') assert page_data["member"].is_a?(Array) end From c6bbf4a50eb8605af1e16863fdd90b5a6296de0d Mon Sep 17 00:00:00 2001 From: MUH <58882014+muhammedBkf@users.noreply.github.com> Date: Thu, 22 May 2025 11:30:26 +0200 Subject: [PATCH 08/23] Feature: Align agents search endpoint with the API response (#149) * Align agents search endpoint with the API response * add helpers functions for agents search * handle failed affiliation parsing * update gemfile.lock * use MultiJson for error handling --- Gemfile.lock | 6 +++++ controllers/search_controller.rb | 7 +++++- helpers/search_helper.rb | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f98ce3f4a..bca2f6111 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -232,6 +232,12 @@ GEM google-protobuf (4.31.0) bigdecimal rake (>= 13) + google-protobuf (4.31.0-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.31.0-x86_64-darwin) + bigdecimal + rake (>= 13) googleapis-common-protos (1.8.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.20) diff --git a/controllers/search_controller.rb b/controllers/search_controller.rb index 5e340f9a6..7661c94f8 100644 --- a/controllers/search_controller.rb +++ b/controllers/search_controller.rb @@ -128,11 +128,16 @@ class SearchController < ApplicationController sort = "score desc, acronym_sort asc, name_sort asc" end - reply 200, search(LinkedData::Models::Agent, + resp = search(LinkedData::Models::Agent, query, fq: fq, qf: qf, page: page, page_size: page_size, sort: sort) + + agents = resp.map { |doc| build_agent_from_search_result(doc) } + + + reply 200, page_object(agents, resp.aggregate) end end diff --git a/helpers/search_helper.rb b/helpers/search_helper.rb index 1e0d2d030..efb0c1def 100644 --- a/helpers/search_helper.rb +++ b/helpers/search_helper.rb @@ -509,6 +509,46 @@ def get_query(params) query = params[:query] || params[:q] query end + def build_agent_from_search_result(doc) + affiliations = Array(doc["affiliations_txt"]).map do |aff_txt| + parse_affiliation(aff_txt) + end.compact + + agent_id = doc["id"].split("/").last + usages = LinkedData::Models::Agent + .find(agent_id) + .include(LinkedData::Models::Agent.attributes) + .first + .usages + + LinkedData::Models::Agent.read_only( + id: doc["id"], + agentType: doc["agentType_t"], + name: doc["name_text"], + homepage: doc["homepage_t"], + acronym: doc["acronym_text"], + email: doc["email_text"], + identifiers: doc["identifiers"], + affiliations: affiliations, + creator: doc["creator_t"], + usages: usages + ) + end + def parse_affiliation(aff_txt) + begin + parsed = MultiJson.load(aff_txt) + LinkedData::Models::Agent.read_only( + id: parsed["id"], + name: parsed["name"], + acronym: parsed["acronym"], + email: parsed["email"], + agentType: parsed["agentType"] + ) + rescue MultiJson::ParseError => e + logger.error "Invalid affiliation JSON: #{aff_txt}" + nil + end + end def search(model, query, params = {}) query = query.blank? ? "*" : query From 9124fe1b791ea9bf853d271fc8c55f2452ca8dd9 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 22 May 2025 11:50:23 +0200 Subject: [PATCH 09/23] use SemanticArtefactDistribution in /distributions route (#153) --- Gemfile.lock | 2 +- controllers/mod/artefacts_controller.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bca2f6111..cec05e614 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 28d4083675acfc852c63198931e8ea5b3edfdfbc + revision: 766c6b0a5f4dd4bad4e11285b818f40195c72524 branch: development specs: ontologies_linked_data (0.0.1) diff --git a/controllers/mod/artefacts_controller.rb b/controllers/mod/artefacts_controller.rb index b29cb12b5..8eb16d03e 100644 --- a/controllers/mod/artefacts_controller.rb +++ b/controllers/mod/artefacts_controller.rb @@ -69,7 +69,7 @@ class ArtefactsController < ApplicationController get '/:artefactID/distributions' do artefact = find_artefact(params["artefactID"]) check_last_modified_segment(LinkedData::Models::SemanticArtefactDistribution, [params["artefactID"]]) - attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactCatalogRecord).first(3) + attributes, page, pagesize= settings_params(LinkedData::Models::SemanticArtefactDistribution).first(3) attributes = LinkedData::Models::SemanticArtefactDistribution.goo_attrs_to_load([]) if includes_param.first == :all distros = artefact.all_distributions(attributes, page, pagesize) reply distros From 73aac00b900b813f8c86a193007f39d7ab4102f5 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 28 May 2025 17:43:28 +0200 Subject: [PATCH 10/23] add virtuoso env variables to use it correctly --- bin/ontoportal | 2 +- docker-compose.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/ontoportal b/bin/ontoportal index f9c7ca18f..5d86cb147 100755 --- a/bin/ontoportal +++ b/bin/ontoportal @@ -119,7 +119,7 @@ provision() { ) for cmd in "${commands[@]}"; do echo "[+] Run: $cmd" - docker_cron_cmd="docker compose -f docker-compose.yml -p ontoportal_docker run --remove-orphans --rm --name cron-service --service-ports ncbo_cron bash -c \"$cmd\" >/dev/null 2>&1" + docker_cron_cmd="docker compose -f docker-compose.yml -p ontoportal_docker run --remove-orphans --rm --name cron-service --service-ports ncbo_cron bash -c \"$cmd\"" if ! eval "$docker_cron_cmd"; then echo "Error: Failed to run provisioning . $cmd" exit 1 diff --git a/docker-compose.yml b/docker-compose.yml index 1cc57b7b7..dc0687f1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,9 @@ x-app: &app GOO_BACKEND_NAME: virtuoso GOO_PORT: 8890 GOO_HOST: virtuoso-ut + GOO_PATH_DATA: /sparql/ + GOO_PATH_QUERY: /sparql/ + GOO_PATH_UPDATE: /sparql/ MGREP_HOST: mgrep-ut MGREP_PORT: 55555 REPOSITORY_FOLDER: /srv/ontoportal/data/repository From 58b0fa292c20fc7e6550f657e031a28ad0879df4 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Mon, 2 Jun 2025 17:53:18 +0200 Subject: [PATCH 11/23] Fix: show federations portals api keys and fix for the API configuration interface (#155) * show federations portals api keys and fix last_modified when updating * allow only admin to see the api keys --- controllers/home_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index ebd32e38c..2fcdf3e51 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -16,7 +16,7 @@ class HomeController < ApplicationController catalog = catalog_class.all.first || create_catalog attributes_to_include = includes_param[0] == :all ? catalog_class.attributes(:all) : catalog_class.goo_attrs_to_load(includes_param) catalog.bring(*attributes_to_include) - catalog.federated_portals = safe_parse(catalog.federated_portals) { |item| item.delete('apikey') } if catalog.loaded_attributes.include?(:federated_portals) + catalog.federated_portals = safe_parse(catalog.federated_portals) { |item| item.delete('apikey') unless current_user&.admin? } if catalog.loaded_attributes.include?(:federated_portals) catalog.fundedBy = safe_parse(catalog.fundedBy) if catalog.loaded_attributes.include?(:fundedBy) reply catalog end @@ -27,6 +27,7 @@ class HomeController < ApplicationController populate_from_params(catalog, params) if catalog.valid? catalog.save + @@root_last_modified = Time.now.httpdate status 200 reply catalog else From 9aa6c92ef2efe04714a7dd60d04450e987ebd779 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Mon, 2 Jun 2025 22:40:59 +0200 Subject: [PATCH 12/23] fix: align development branch with development docker image (#156) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 564fc8d2d..3d11fc88b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ x-app: &app - image: agroportal/ontologies_api:master + image: agroportal/ontologies_api:development environment: &env # default bundle config resolves to /usr/local/bundle/config inside of the container # we are setting it to local app directory if we need to use 'bundle config local' From cd9b79d417751692f6dc524dbe60101093ec962f Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 4 Jun 2025 12:34:30 +0200 Subject: [PATCH 13/23] use check_last_modified to handle last modified and caching the catalog (#157) --- controllers/home_controller.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 2fcdf3e51..177a7c8de 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -9,11 +9,10 @@ class HomeController < ApplicationController default_responses(success: true) end get do - expires 3600, :public - last_modified @@root_last_modified ||= Time.now.httpdate - catalog_class = LinkedData::Models::SemanticArtefactCatalog catalog = catalog_class.all.first || create_catalog + check_last_modified(catalog) + attributes_to_include = includes_param[0] == :all ? catalog_class.attributes(:all) : catalog_class.goo_attrs_to_load(includes_param) catalog.bring(*attributes_to_include) catalog.federated_portals = safe_parse(catalog.federated_portals) { |item| item.delete('apikey') unless current_user&.admin? } if catalog.loaded_attributes.include?(:federated_portals) @@ -27,7 +26,6 @@ class HomeController < ApplicationController populate_from_params(catalog, params) if catalog.valid? catalog.save - @@root_last_modified = Time.now.httpdate status 200 reply catalog else From 7aeee27e037259a2064241cbd2ac6bd4be1b1ea1 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 4 Jun 2025 15:26:30 +0200 Subject: [PATCH 14/23] add security check when updating the catalog --- controllers/home_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/home_controller.rb b/controllers/home_controller.rb index 177a7c8de..6df9b4c78 100644 --- a/controllers/home_controller.rb +++ b/controllers/home_controller.rb @@ -21,6 +21,7 @@ class HomeController < ApplicationController end patch do + error 401, "Unauthorized: Admin access required to update the catalog" unless current_user&.admin? catalog = LinkedData::Models::SemanticArtefactCatalog.where.first error 422, "There is no catalog configs in the triple store" if catalog.nil? populate_from_params(catalog, params) From 2967f7bb4a02f7310b8e0ee7b64f78e65a57fc2e Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 25 Jun 2025 17:30:44 +0200 Subject: [PATCH 15/23] update Gemfile.lock --- Gemfile.lock | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index cec05e614..e4d53da23 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 766c6b0a5f4dd4bad4e11285b818f40195c72524 + revision: eb4b482e9e973b1ff0dfad3b40f5ace74134fa94 branch: development specs: ontologies_linked_data (0.0.1) @@ -146,14 +146,14 @@ GEM sshkit (>= 1.6.1, != 1.7.0) ansi (1.5.0) ast (2.4.3) - base64 (0.2.0) + base64 (0.3.0) bcp47_spec (0.2.1) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) bcrypt_pbkdf (1.1.1-x86_64-darwin) - benchmark (0.4.0) - bigdecimal (3.1.9) + benchmark (0.4.1) + bigdecimal (3.2.2) builder (3.3.0) capistrano (3.19.2) airbrussh (>= 1.0.0) @@ -185,7 +185,7 @@ GEM faraday-net_http (>= 2.0, < 3.5) json logger - faraday-net_http (3.4.0) + faraday-net_http (3.4.1) net-http (>= 0.5.0) faraday-retry (2.3.1) faraday (~> 2.0) @@ -276,7 +276,7 @@ GEM mutex_m i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.12.0) + json (2.12.2) json-canonicalization (0.4.0) json-ld (3.2.5) htmlentities (~> 4.3) @@ -307,7 +307,7 @@ GEM mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2025.0520) + mime-types-data (3.2025.0624) mini_mime (1.1.5) minitest (5.25.5) minitest-fail-fast (0.1.0) @@ -331,9 +331,9 @@ GEM time net-http (0.6.0) uri - net-http-persistent (4.0.5) - connection_pool (~> 2.2) - net-imap (0.5.8) + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-imap (0.5.9) date net-protocol net-pop (0.1.2) @@ -349,13 +349,13 @@ GEM net-ssh (7.3.0) netrc (0.11.0) newrelic_rpm (9.19.0) - oj (3.16.10) + oj (3.16.11) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) logger os (1.1.4) - ostruct (0.6.1) + ostruct (0.6.2) pandoc-ruby (2.1.10) parallel (1.27.0) parseconfig (1.1.2) @@ -371,7 +371,7 @@ GEM public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.15) + rack (3.1.16) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -396,10 +396,12 @@ GEM rainbow (3.1.1) raindrops (0.20.1) rake (13.2.1) - rdf (3.3.2) + rdf (3.3.3) bcp47_spec (~> 0.2) bigdecimal (~> 3.1, >= 3.1.5) link_header (~> 0.0, >= 0.0.8) + logger (~> 1.5) + ostruct (~> 0.6) rdf-rdfxml (3.3.0) builder (~> 3.2, >= 3.2.4) htmlentities (~> 4.3) @@ -412,7 +414,7 @@ GEM rexml (~> 3.2) redis (5.4.0) redis-client (>= 0.22.0) - redis-client (0.24.0) + redis-client (0.25.0) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) From 5df2318ff99a6a7e27f7965f8ff5c83f6d9cef09 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Fri, 27 Jun 2025 10:50:06 +0200 Subject: [PATCH 16/23] update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e4d53da23..f57b1a8d9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: eb4b482e9e973b1ff0dfad3b40f5ace74134fa94 + revision: 8ce21d045c6fe829c6893c472e2293233d65d402 branch: development specs: ontologies_linked_data (0.0.1) From 858acf6a4e7e53381391e64fdc7587ea4f1c91cb Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Wed, 2 Jul 2025 17:49:49 +0200 Subject: [PATCH 17/23] address QA changes for MOD-API --- Gemfile.lock | 4 ++-- test/controllers/test_mod_api_controller.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f57b1a8d9..cf1f5ff8e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 8ce21d045c6fe829c6893c472e2293233d65d402 + revision: 505fbb9f2d9ec23834cfcd5ffd2ba1249b2b254e branch: development specs: ontologies_linked_data (0.0.1) @@ -307,7 +307,7 @@ GEM mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2025.0624) + mime-types-data (3.2025.0701) mini_mime (1.1.5) minitest (5.25.5) minitest-fail-fast (0.1.0) diff --git a/test/controllers/test_mod_api_controller.rb b/test/controllers/test_mod_api_controller.rb index 2ae052322..4bb637e39 100644 --- a/test/controllers/test_mod_api_controller.rb +++ b/test/controllers/test_mod_api_controller.rb @@ -121,7 +121,7 @@ def test_latest_distribution get route assert last_response.ok? dist_data = MultiJson.load(last_response.body) - assert_equal 1, dist_data["distributionId"] + assert_equal 2, dist_data["distributionId"] end def test_resources From a968748c352dd45d3c6502ecca56ed4ab1c72fa2 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 3 Jul 2025 10:38:33 +0200 Subject: [PATCH 18/23] update Gemfile.lock --- Gemfile.lock | 52 ++++++++++++++++------------------------------------ 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c0abccc4a..63662f12f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -103,7 +103,7 @@ GIT GIT remote: https://github.com/sinatra/sinatra.git - revision: 91cfb548c9e50a65324a9ce9e4ea5f10cd897027 + revision: 5e1598501eb23a8673d61034df7be7d50c228400 specs: rack-protection (4.1.1) base64 (>= 0.1.0) @@ -150,8 +150,6 @@ GEM bcp47_spec (0.2.1) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) - bcrypt_pbkdf (1.1.1-arm64-darwin) - bcrypt_pbkdf (1.1.1-x86_64-darwin) benchmark (0.4.1) bigdecimal (3.2.2) builder (3.3.0) @@ -187,7 +185,7 @@ GEM logger faraday-net_http (3.4.1) net-http (>= 0.5.0) - faraday-retry (2.3.1) + faraday-retry (2.3.2) faraday (~> 2.0) ffi (1.15.5) fugit (1.11.1) @@ -213,7 +211,7 @@ GEM google-cloud-errors (~> 1.0) google-apis-analytics_v3 (0.17.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.17.0) + google-apis-core (0.18.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.3, < 3.a) @@ -224,18 +222,12 @@ GEM google-cloud-core (1.8.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.3.0) + google-cloud-env (2.3.1) base64 (~> 0.2) faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) google-logging-utils (0.2.0) - google-protobuf (4.31.0) - bigdecimal - rake (>= 13) - google-protobuf (4.31.0-arm64-darwin) - bigdecimal - rake (>= 13) - google-protobuf (4.31.0-x86_64-darwin) + google-protobuf (4.30.2-x86_64-linux) bigdecimal rake (>= 13) googleapis-common-protos (1.8.0) @@ -252,16 +244,7 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.72.0) - google-protobuf (>= 3.25, < 5.0) - googleapis-common-protos-types (~> 1.0) - grpc (1.72.0-arm64-darwin) - google-protobuf (>= 3.25, < 5.0) - googleapis-common-protos-types (~> 1.0) - grpc (1.72.0-x86_64-darwin) - google-protobuf (>= 3.25, < 5.0) - googleapis-common-protos-types (~> 1.0) - grpc (1.72.0-x86_64-linux) + grpc (1.73.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -288,7 +271,7 @@ GEM json-schema (5.1.1) addressable (~> 2.8) bigdecimal (~> 3.1) - jwt (2.10.1) + jwt (2.10.2) base64 kgio (2.11.4) language_server-protocol (3.17.0.5) @@ -348,7 +331,6 @@ GEM net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.19.0) oj (3.16.11) bigdecimal (>= 3.0) ostruct (>= 0.2) @@ -371,7 +353,7 @@ GEM public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.16) + rack (3.1.10) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -383,7 +365,7 @@ GEM rack-cors (3.0.0) logger rack (>= 3.0.14) - rack-mini-profiler (3.3.1) + rack-mini-profiler (4.0.0) rack (>= 1.2.0) rack-session (2.1.1) base64 (>= 0.1.0) @@ -395,7 +377,7 @@ GEM rack (>= 3) rainbow (3.1.1) raindrops (0.20.1) - rake (13.2.1) + rake (13.3.0) rdf (3.3.3) bcp47_spec (~> 0.2) bigdecimal (~> 3.1, >= 3.1.5) @@ -438,7 +420,7 @@ GEM rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.75.6) + rubocop (1.77.0) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -446,10 +428,10 @@ GEM parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.44.0, < 2.0) + rubocop-ast (>= 1.45.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.44.1) + rubocop-ast (1.45.1) parser (>= 3.3.7.2) prism (~> 1.4) ruby-progressbar (1.13.0) @@ -459,6 +441,9 @@ GEM rufus-scheduler (3.9.2) fugit (~> 1.1, >= 1.11.1) securerandom (0.4.1) + sentry-ruby (5.26.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) signet (0.20.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) @@ -516,11 +501,6 @@ GEM webrick (1.9.1) PLATFORMS - aarch64-linux - arm64-darwin - ruby - x86-linux - x86_64-darwin x86_64-linux DEPENDENCIES From aebbc4870b15ee22011a384e57a290560527dcf1 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 3 Jul 2025 17:41:32 +0200 Subject: [PATCH 19/23] update gems versions --- Gemfile | 7 +++---- Gemfile.lock | 30 +++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 81d24a342..efd97e1f2 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ gem 'json-schema' gem 'multi_json' gem 'oj' gem 'parseconfig' -gem 'rack', '3.1.10' +gem 'rack' gem 'rake' gem 'rexml' # Investigate why unicorn fails to start under ruby 3 without adding rexml gem to the Gemfile gem 'sinatra' @@ -17,7 +17,7 @@ end gem 'request_store' gem 'parallel' -gem 'google-protobuf', '~> 4.30.0' +gem 'google-protobuf' gem 'net-ftp' gem 'json-ld', '~> 3.2.0' gem 'rdf-raptor', github:'ruby-rdf/rdf-raptor', ref: '6392ceabf71c3233b0f7f0172f662bd4a22cd534' # use version 3.3.0 when available @@ -90,5 +90,4 @@ group :test do gem 'simplecov-cobertura' # for codecov.io gem 'webmock' gem 'webrick' -end - +end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 63662f12f..50575ee1c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -29,7 +29,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: 3cf75117e8023115cfdec1c8d5a3d78f273d19db + revision: df22084bd5960254cc21408f1090a7faf9e3ab72 branch: master specs: ncbo_cron (0.0.1) @@ -150,6 +150,8 @@ GEM bcp47_spec (0.2.1) bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) + bcrypt_pbkdf (1.1.1-arm64-darwin) + bcrypt_pbkdf (1.1.1-x86_64-darwin) benchmark (0.4.1) bigdecimal (3.2.2) builder (3.3.0) @@ -227,7 +229,13 @@ GEM faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) google-logging-utils (0.2.0) - google-protobuf (4.30.2-x86_64-linux) + google-protobuf (4.31.1) + bigdecimal + rake (>= 13) + google-protobuf (4.31.1-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.31.1-x86_64-darwin) bigdecimal rake (>= 13) googleapis-common-protos (1.8.0) @@ -244,6 +252,15 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) + grpc (1.73.0) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) + grpc (1.73.0-arm64-darwin) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) + grpc (1.73.0-x86_64-darwin) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) grpc (1.73.0-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) @@ -353,7 +370,7 @@ GEM public_suffix (6.0.2) raabro (1.4.0) racc (1.8.1) - rack (3.1.10) + rack (3.1.16) rack-accept (0.4.5) rack (>= 0.4) rack-attack (6.7.0) @@ -501,6 +518,9 @@ GEM webrick (1.9.1) PLATFORMS + arm64-darwin + ruby + x86_64-darwin x86_64-linux DEPENDENCIES @@ -515,7 +535,7 @@ DEPENDENCIES ed25519 (>= 1.2, < 2.0) ffi (~> 1.15.0) goo! - google-protobuf (~> 4.30.0) + google-protobuf haml (~> 5.2.2) json-ld (~> 3.2.0) json-schema @@ -534,7 +554,7 @@ DEPENDENCIES pandoc-ruby parallel parseconfig - rack (= 3.1.10) + rack rack-accept rack-attack rack-cache From d99a2448a13028fbde278de06c4da7445cdc0ba8 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 3 Jul 2025 17:51:04 +0200 Subject: [PATCH 20/23] user rack and lock protobuf version --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index efd97e1f2..ac4f7e12f 100644 --- a/Gemfile +++ b/Gemfile @@ -17,7 +17,7 @@ end gem 'request_store' gem 'parallel' -gem 'google-protobuf' +gem 'google-protobuf', '~> 4.30.0' gem 'net-ftp' gem 'json-ld', '~> 3.2.0' gem 'rdf-raptor', github:'ruby-rdf/rdf-raptor', ref: '6392ceabf71c3233b0f7f0172f662bd4a22cd534' # use version 3.3.0 when available From 792071fa9833b2bcf06008dcd3476f99f31da6a8 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 3 Jul 2025 17:55:11 +0200 Subject: [PATCH 21/23] update gemfile.lock --- Gemfile.lock | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 50575ee1c..60a8c83d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -229,13 +229,22 @@ GEM faraday (>= 1.0, < 3.a) google-cloud-errors (1.5.0) google-logging-utils (0.2.0) - google-protobuf (4.31.1) + google-protobuf (4.30.2) bigdecimal rake (>= 13) - google-protobuf (4.31.1-arm64-darwin) + google-protobuf (4.30.2-aarch64-linux) bigdecimal rake (>= 13) - google-protobuf (4.31.1-x86_64-darwin) + google-protobuf (4.30.2-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.30.2-x86-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.30.2-x86_64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.30.2-x86_64-linux) bigdecimal rake (>= 13) googleapis-common-protos (1.8.0) @@ -255,9 +264,15 @@ GEM grpc (1.73.0) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) + grpc (1.73.0-aarch64-linux) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) grpc (1.73.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) + grpc (1.73.0-x86-linux) + google-protobuf (>= 3.25, < 5.0) + googleapis-common-protos-types (~> 1.0) grpc (1.73.0-x86_64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) @@ -518,8 +533,10 @@ GEM webrick (1.9.1) PLATFORMS + aarch64-linux arm64-darwin ruby + x86-linux x86_64-darwin x86_64-linux @@ -535,7 +552,7 @@ DEPENDENCIES ed25519 (>= 1.2, < 2.0) ffi (~> 1.15.0) goo! - google-protobuf + google-protobuf (~> 4.30.0) haml (~> 5.2.2) json-ld (~> 3.2.0) json-schema From e069d461c4a75ec9d9514a6c3991024d96fe7ea6 Mon Sep 17 00:00:00 2001 From: Imad Bourouche Date: Thu, 3 Jul 2025 23:51:30 +0200 Subject: [PATCH 22/23] fix: remove Rack::RewindableInput Middleware causing --- app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.rb b/app.rb index d7d4c3150..1c8b5c3e9 100644 --- a/app.rb +++ b/app.rb @@ -145,7 +145,7 @@ config.send_default_pii = true end - use Rack::RewindableInput::Middleware + # use Rack::RewindableInput::Middleware use Sentry::Rack::CaptureExceptions end From f9ecc4c409d6cd47cbaed66d00d63187cf974172 Mon Sep 17 00:00:00 2001 From: Guillaume Alviset Date: Thu, 17 Jul 2025 16:44:10 +0200 Subject: [PATCH 23/23] Updated Gemfile --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index bb060ae6b..183344ca2 100644 --- a/Gemfile +++ b/Gemfile @@ -53,7 +53,7 @@ gem 'pandoc-ruby' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'development' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ontoportal-lirmm/ncbo_ontology_recommender.git', branch: 'development' -gem 'ontologies_linked_data', github: 'earthportal/ontologies_linked_data', branch: 'feature/projects' +gem 'ontologies_linked_data', github: 'earthportal/ontologies_linked_data', branch: 'development' gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'