diff --git a/.github/workflows/testkit-unit-tests.yml b/.github/workflows/testkit-unit-tests.yml new file mode 100644 index 00000000..2f3e11ad --- /dev/null +++ b/.github/workflows/testkit-unit-tests.yml @@ -0,0 +1,87 @@ +name: Docker Unit Tests + +on: + push: + branches: + - '**' + tags-ignore: + - '**' + pull_request: + +env: + # CI execution mode for backend tests: + # - container: run `test:docker::container` (default) + # - native: run `test:docker:` on host Ruby + OPTK_CI_RUN_MODE: ${{ vars.OPTK_CI_RUN_MODE || 'container' }} + # Example override to force native mode in this workflow file: + # OPTK_CI_RUN_MODE: native + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + backends: ${{ steps.cfg.outputs.backends }} + steps: + - uses: actions/checkout@v6 + + - id: cfg + name: Read backend matrix from .ontoportal-testkit.yml + run: | + BACKENDS=$(ruby -ryaml -rjson -e 'c=YAML.safe_load_file(".ontoportal-testkit.yml") || {}; b=c["backends"] || %w[fs ag vo gd]; puts JSON.generate(b)') + echo "backends=$BACKENDS" >> "$GITHUB_OUTPUT" + + test: + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + backend: ${{ fromJson(needs.prepare.outputs.backends) }} + + steps: + - uses: actions/checkout@v6 + + - name: Set up Ruby from .ruby-version + uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - name: Set up Java 11 (native mode) + if: env.OPTK_CI_RUN_MODE == 'native' + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '11' + + - name: Install native system dependencies + if: env.OPTK_CI_RUN_MODE == 'native' + run: | + sudo apt-get update + sudo apt-get install -y raptor2-utils + + - name: Run unit tests + env: + CI: "true" + TESTOPTS: "--verbose" + BACKEND: ${{ matrix.backend }} + run: | + MODE="${OPTK_CI_RUN_MODE:-container}" + TASK="test:docker:${BACKEND}" + if [ "$MODE" = "container" ]; then + TASK="${TASK}:container" + elif [ "$MODE" != "native" ]; then + echo "Invalid OPTK_CI_RUN_MODE=$MODE (expected container or native)" + exit 1 + fi + + bundle exec rake "$TASK" + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: unittests,${{ matrix.backend }} + verbose: true + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index f887556d..f828b66e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.tags* .bundle .config +.serena coverage InstalledFiles lib/bundler/man @@ -25,3 +26,8 @@ doc/ .idea/* projectFilesBackup/* + +config/config.rb +queries.txt + +*.iml diff --git a/.ontoportal-testkit.yml b/.ontoportal-testkit.yml new file mode 100644 index 00000000..75588b98 --- /dev/null +++ b/.ontoportal-testkit.yml @@ -0,0 +1,8 @@ +component_name: goo +app_service: test-container +backends: + - fs + - ag + - vo + - gd +dependency_services: [] diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..f15386a5 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.2.10 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6c566230 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +ARG RUBY_VERSION=3.2 +ARG DISTRO=bullseye +ARG TESTKIT_BASE_IMAGE=ontoportal/testkit-base:ruby${RUBY_VERSION}-${DISTRO} +FROM ${TESTKIT_BASE_IMAGE} + +WORKDIR /app + +COPY Gemfile* *.gemspec ./ + +# Respect the project's Bundler lock when present. +RUN if [ -f Gemfile.lock ]; then \ + BUNDLER_VERSION=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | tr -d ' '); \ + gem install bundler -v "$BUNDLER_VERSION"; \ + fi + +RUN bundle install --jobs 4 --retry 3 + +COPY . ./ + +CMD ["bundle", "exec", "rake"] diff --git a/Gemfile b/Gemfile index 30167e35..e7ee86c9 100644 --- a/Gemfile +++ b/Gemfile @@ -2,20 +2,26 @@ source 'https://rubygems.org' gemspec -gem 'rake' -gem 'pry' -gem 'simplecov' -gem 'minitest', '< 5.0' -gem 'activesupport' +gem "activesupport" +gem "rake" gem "uuid" -gem 'cube-ruby', require: "cube" -gem 'faraday', '~> 1.9' +gem "request_store" + +group :test do + gem "minitest", '< 5.0' + gem "pry" + gem 'simplecov' + gem 'simplecov-cobertura' # for submitting code coverage results to codecov.io + gem 'ontoportal_testkit', github: 'alexskr/ontoportal_testkit', branch: 'main' +end group :profiling do - gem 'sinatra' - gem 'rack-accept' - gem 'rack-post-body-to-params' - gem 'thin' + gem "rack-accept" + gem "rack-post-body-to-params" + gem "sinatra" + gem "thin" end -gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'ontoportal-lirmm-development' +gem "rdf-raptor", github: "ruby-rdf/rdf-raptor", ref: "6392ceabf71c3233b0f7f0172f662bd4a22cd534" # use version 3.3.0 when available +gem 'net-ftp' diff --git a/Gemfile.lock b/Gemfile.lock index 2945f1d3..67b81dae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,25 +1,43 @@ GIT - remote: https://github.com/ontoportal-lirmm/sparql-client.git - revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0 - branch: master + remote: https://github.com/alexskr/ontoportal_testkit.git + revision: 45ecfaf2a621b670dad373cd35c1c4b1b529fa96 + branch: main specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) + ontoportal_testkit (0.1.0) + rake (>= 13.0) + +GIT + remote: https://github.com/ncbo/sparql-client.git + revision: 2ac20b217bb7ad2b11305befe0ee77d75e44eac5 + branch: ontoportal-lirmm-development + specs: + sparql-client (3.2.2) + net-http-persistent (~> 4.0, >= 4.0.2) + rdf (~> 3.2, >= 3.2.11) + +GIT + remote: https://github.com/ruby-rdf/rdf-raptor.git + revision: 6392ceabf71c3233b0f7f0172f662bd4a22cd534 + ref: 6392ceabf71c3233b0f7f0172f662bd4a22cd534 + specs: + rdf-raptor (3.3.0) + ffi (~> 1.15) + rdf (~> 3.3) PATH remote: . specs: goo (0.0.2) - addressable (= 2.3.5) + addressable (~> 2.8) pry - rdf (= 1.0.8) + rdf + rdf-raptor + rdf-rdfxml + rdf-vocab redis rest-client rsolr sparql-client - systemu uuid GEM @@ -31,82 +49,115 @@ GEM multi_json (~> 1.3) thread_safe (~> 0.1) tzinfo (~> 0.3.37) - addressable (2.3.5) - builder (3.2.4) + addressable (2.8.9) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + bcp47_spec (0.2.1) + bigdecimal (3.3.1) + builder (3.3.0) coderay (1.1.3) - concurrent-ruby (1.1.10) - connection_pool (2.3.0) - cube-ruby (0.0.3) + concurrent-ruby (1.3.6) + connection_pool (3.0.2) daemons (1.4.1) - docile (1.4.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) + date (3.5.1) + docile (1.4.1) + domain_name (0.6.20240107) eventmachine (1.2.7) - faraday (1.10.3) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) - ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) + faraday (2.14.1) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.2) + net-http (~> 0.5) + ffi (1.17.3) + ffi (1.17.3-aarch64-linux-gnu) + ffi (1.17.3-arm64-darwin) + ffi (1.17.3-x86_64-darwin) + ffi (1.17.3-x86_64-linux-gnu) + htmlentities (4.4.2) http-accept (1.7.0) - http-cookie (1.0.5) + http-cookie (1.1.0) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - json_pure (2.6.3) + io-console (0.8.2) + json (2.19.1) + link_header (0.0.8) + logger (1.7.0) macaddr (1.7.2) systemu (~> 2.6.5) - method_source (1.0.0) - mime-types (3.4.1) - mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) + method_source (1.1.0) + mime-types (3.7.0) + logger + mime-types-data (~> 3.2025, >= 3.2025.0507) + mime-types-data (3.2026.0303) minitest (4.7.5) - multi_json (1.15.0) - multipart-post (2.2.3) - mustermann (3.0.0) + multi_json (1.19.1) + mustermann (3.0.4) ruby2_keywords (~> 0.0.1) - net-http-persistent (2.9.4) + net-ftp (0.3.9) + net-protocol + time + net-http (0.9.1) + uri (>= 0.11.1) + net-http-persistent (4.0.8) + connection_pool (>= 2.2.4, < 4) + net-protocol (0.2.2) + timeout netrc (0.11.0) - pry (0.14.2) + ostruct (0.6.3) + pry (0.16.0) coderay (~> 1.1) method_source (~> 1.0) - rack (2.2.6.2) + reline (>= 0.6.0) + public_suffix (7.0.5) + rack (3.2.5) rack-accept (0.4.5) rack (>= 0.4) rack-post-body-to-params (0.1.8) activesupport (>= 2.3) - rack-protection (3.0.5) - rack - rake (13.0.6) - rdf (1.0.8) - addressable (>= 2.2) - redis (5.0.6) - redis-client (>= 0.9.0) - redis-client (0.12.1) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rake (13.3.1) + rdf (3.3.4) + bcp47_spec (~> 0.2) + bigdecimal (~> 3.1, >= 3.1.5) + link_header (~> 0.0, >= 0.0.8) + logger (~> 1.5) + ostruct (~> 0.6) + readline (~> 0.0) + rdf-rdfxml (3.3.0) + builder (~> 3.2, >= 3.2.4) + htmlentities (~> 4.3) + rdf (~> 3.3) + rdf-xsd (~> 3.3) + rdf-vocab (3.3.3) + rdf (~> 3.3) + rdf-xsd (3.3.0) + rdf (~> 3.3) + rexml (~> 3.2) + readline (0.0.4) + reline + redis (5.4.1) + redis-client (>= 0.22.0) + redis-client (0.27.0) connection_pool + reline (0.6.3) + io-console (~> 0.5) + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rsolr (2.5.0) + rexml (3.4.4) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby2_keywords (0.0.5) @@ -114,46 +165,59 @@ GEM docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) - simplecov-html (0.12.3) + simplecov-cobertura (3.1.0) + rexml + simplecov (~> 0.19) + simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) - sinatra (3.0.5) + sinatra (4.2.1) + logger (>= 1.6.0) mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.0.5) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) systemu (2.6.5) - thin (1.8.1) + thin (2.0.1) daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) - rack (>= 1, < 3) + logger + rack (>= 1, < 4) thread_safe (0.3.6) - tilt (2.0.11) - tzinfo (0.3.61) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) + tilt (2.7.0) + time (0.4.2) + date + timeout (0.6.1) + tzinfo (0.3.62) + uri (1.1.1) uuid (2.3.9) macaddr (~> 1.0) PLATFORMS + aarch64-linux-gnu + arm64-darwin ruby - x86_64-darwin-16 + x86_64-darwin + x86_64-linux-gnu DEPENDENCIES activesupport - cube-ruby - faraday (~> 1.9) goo! minitest (< 5.0) + net-ftp + ontoportal_testkit! pry rack-accept rack-post-body-to-params rake + rdf-raptor! + request_store simplecov + simplecov-cobertura sinatra sparql-client! thin uuid BUNDLED WITH - 2.1.4 + 4.0.7 diff --git a/README.md b/README.md index b1553b79..3ea16b6e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,42 @@ To see Goo in action browse to the following links: - [BioPortal New API Documentation](http://stagedata.bioontology.org/documentation) - [BioPortal New API Access](http://stagedata.bioontology.org/) +## Tests + +Run the unit tests: + +``` +bundle exec rake test +``` + +Run docker-backed tests (host Ruby): + +``` +bundle exec rake test:docker:fs +bundle exec rake test:docker:ag +bundle exec rake test:docker:vo +bundle exec rake test:docker:gd +``` + +Run docker-backed tests inside the Linux container: + +``` +bundle exec rake test:docker:fs:linux +bundle exec rake test:docker:ag:linux +bundle exec rake test:docker:vo:linux +bundle exec rake test:docker:gd:linux +``` + +Start a shell in the Linux test container (default backend `fs`): + +``` +bundle exec rake test:docker:shell +bundle exec rake test:docker:shell[ag] +``` + +Set `OP_KEEP_CONTAINERS=1` to keep services up after tests or shell exit. +Set `OP_TEST_DOCKER_BACKEND=ag` (or `fs`, `vo`, `gd`) to change the default backend for `test:docker:shell`, `test:docker:up`, and `test:docker:down`. + ## Schema Definitions (DSL) diff --git a/Rakefile b/Rakefile index 89da7e61..80c18410 100644 --- a/Rakefile +++ b/Rakefile @@ -1,56 +1,106 @@ require 'rake/testtask' +task default: %w[test] + Rake::TestTask.new do |t| t.libs = [] t.test_files = FileList['test/test*.rb'].select { |x| !x["index"] } + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:persistent" - t.test_files = FileList['test/test_model_persistence.rb'] + t.name = "test:persistence" + t.test_files = FileList['test/test_basic_persistence.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:person" - t.test_files = FileList['test/test_model_person.rb'] + t.name = "test:cache" + t.test_files = FileList['test/test_cache.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:anon" - t.test_files = FileList['test/test_model_unnamed.rb'] + t.name = "test:chunks_write" + t.test_files = FileList['test/test_chunks_write.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:review" - t.test_files = FileList['test/test_review.rb'] + t.name = "test:collections" + t.test_files = FileList['test/test_collections.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:dependent" - t.test_files = FileList['test/test_model_dependent.rb'] + t.name = "test:dsl_settings" + t.test_files = FileList['test/test_dsl_settings.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "test:find" - t.test_files = FileList['test/test_model_find.rb'] + t.name = "test:enum" + t.test_files = FileList['test/test_enum.rb'] + t.warning = false end Rake::TestTask.new do |t| - t.libs = [] - t.name = "benchmark" - t.test_files = FileList['test/benchmark.rb'] + t.name = "test:index" + t.test_files = FileList['test/test_index.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:inverse" + t.test_files = FileList['test/test_inverse.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:model_complex" + t.test_files = FileList['test/test_model_complex.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:name_with" + t.test_files = FileList['test/test_name_with.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:namespaces" + t.test_files = FileList['test/test_namespaces.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:read_only" + t.test_files = FileList['test/test_read_only.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:schemaless" + t.test_files = FileList['test/test_schemaless.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:search" + t.test_files = FileList['test/test_search.rb'] + t.warning = false +end + +Rake::TestTask.new do |t| + t.name = "test:where" + t.test_files = FileList['test/test_where.rb'] + t.warning = false end desc "Console for working with data" task :console do require_relative "test/test_case" - GooTest.configure_goo binding.pry end diff --git a/config/config.rb.sample b/config/config.rb.sample new file mode 100644 index 00000000..edac79df --- /dev/null +++ b/config/config.rb.sample @@ -0,0 +1,23 @@ +Goo.config do |config| + # 4store + config.goo_backend_name = '4store' + config.goo_port = 9000 + config.goo_host = 'localhost' + config.goo_path_query = '/sparql/' + config.goo_path_data = '/data/' + config.goo_path_update = '/update/' + + # AllegroGraph + # config.goo_backend_name = 'allegrograph' + # config.goo_port = 10035 + # config.goo_host = 'localhost' + # config.goo_path_query = "/repositories/ontoportal" + # config.goo_path_data = "/repositories/ontoportal/statements/" + # config.goo_path_update = "/repositories/ontoportal/statements/" + + config.search_server_url = 'http://localhost:8983/solr/' + config.goo_redis_host = 'localhost' + config.goo_redis_port = 6379 + config.bioportal_namespace = 'http://data.bioontology.org/' + config.queries_debug = false +end diff --git a/config/config.test.rb b/config/config.test.rb new file mode 100644 index 00000000..972ad2da --- /dev/null +++ b/config/config.test.rb @@ -0,0 +1,2 @@ +Goo.config do |config| +end diff --git a/goo.gemspec b/goo.gemspec index b96505fc..b053e72a 100644 --- a/goo.gemspec +++ b/goo.gemspec @@ -1,19 +1,20 @@ Gem::Specification.new do |s| - s.name = 'goo' - s.version = '0.0.2' - s.date = '2012-11-21' - s.summary = "" + s.name = "goo" + s.version = "0.0.2" + s.summary = "Graph Oriented Objects (GOO) for Ruby. A RDF/SPARQL based ORM." s.authors = ["Manuel Salvadores", "Paul Alexander"] - s.email = 'manuelso@stanford.edu' - s.files = Dir['lib/**/*.rb'] - s.homepage = 'http://github.com/ncbo/goo' - s.add_dependency("uuid") - s.add_dependency("systemu") # remove when https://github.com/ahoward/macaddr/pull/20 is resolved + s.email = "manuelso@stanford.edu" + s.files = Dir["lib/**/*.rb"] + s.homepage = "http://github.com/ncbo/goo" + s.add_dependency("addressable", "~> 2.8") + s.add_dependency("pry") + s.add_dependency("rdf") + s.add_dependency("rdf-vocab") + s.add_dependency("rdf-rdfxml") + s.add_dependency("rdf-raptor") + s.add_dependency("redis") + s.add_dependency("rest-client") s.add_dependency("rsolr") - s.add_dependency("rdf", "= 1.0.8") s.add_dependency("sparql-client") - s.add_dependency("addressable", "= 2.3.5") - s.add_dependency("rest-client") - s.add_dependency("redis") - s.add_dependency("pry") + s.add_dependency("uuid") end diff --git a/lib/goo.rb b/lib/goo.rb index 1867c7d6..59441811 100644 --- a/lib/goo.rb +++ b/lib/goo.rb @@ -1,6 +1,8 @@ require "pry" require "rdf" +require "rdf/vocab" require "rdf/ntriples" +require "rdf/rdfxml" require "sparql/client" require "set" @@ -10,12 +12,16 @@ require 'rest_client' require 'redis' require 'uuid' -require "cube" +require_relative "goo/config/config" require_relative "goo/sparql/sparql" require_relative "goo/search/search" require_relative "goo/base/base" require_relative "goo/validators/enforce" +require_relative "goo/validators/validator" +project_root = File.dirname(File.absolute_path(__FILE__)) +Dir.glob("#{project_root}/goo/validators/implementations/*", &method(:require)) + require_relative "goo/utils/utils" require_relative "goo/mixins/sparql_client" @@ -25,25 +31,77 @@ module Goo @@resource_options = Set.new([:persistent]).freeze + # Define the languages from which the properties values will be taken + # It choose the first language that match otherwise return all the values + @@main_languages = %w[en] + @@requested_language = nil + @@configure_flag = false @@sparql_backends = {} @@model_by_name = {} @@search_backends = {} @@search_connection = {} + @@search_collections = {} @@default_namespace = nil @@id_prefix = nil @@redis_client = nil - @@cube_options = nil @@namespaces = {} @@pluralize_models = false @@uuid = UUID.new @@debug_enabled = false @@use_cache = false - + @@query_logging = false + @@query_logging_file = './queries.log' @@slice_loading_size = 500 + + + def self.log_debug_file(str) + debug_file = "./queries.txt" + File.write(debug_file, str.to_s + "\n", mode: 'a') + end + + + + def backend_4s? + sparql_backend_name.downcase.eql?("4store") + end + + def backend_ag? + sparql_backend_name.downcase.eql?("allegrograph") + end + + def backend_gb? + sparql_backend_name.downcase.eql?("graphdb") + end + + def backend_vo? + sparql_backend_name.downcase.eql?("virtuoso") + end + + + def self.main_languages + @@main_languages + end + def self.main_languages=(lang) + @@main_languages = lang + end + + def self.requested_language + @@requested_language + end + + def self.requested_language=(lang) + @@requested_language = lang + end + + def self.language_includes(lang) + lang_str = lang.to_s + main_languages.index { |l| lang_str.downcase.eql?(l) || lang_str.upcase.eql?(l)} + end + def self.add_namespace(shortcut, namespace,default=false) - if !(namespace.instance_of? RDF::Vocabulary) + unless namespace.instance_of? RDF::Vocabulary raise ArgumentError, "Namespace must be a RDF::Vocabulary object" end @@namespaces[shortcut.to_sym] = namespace @@ -58,36 +116,35 @@ def self.add_sparql_backend(name, *opts) opts = opts[0] @@sparql_backends = @@sparql_backends.dup @@sparql_backends[name] = opts - @@sparql_backends[name][:query]=Goo::SPARQL::Client.new(opts[:query], - {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", - read_timeout: 10000, - validate: false, - redis_cache: @@redis_client, - cube_options: @@cube_options}) - @@sparql_backends[name][:update]=Goo::SPARQL::Client.new(opts[:update], - {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", - read_timeout: 10000, - validate: false, - redis_cache: @@redis_client, - cube_options: @@cube_options}) - @@sparql_backends[name][:data]=Goo::SPARQL::Client.new(opts[:data], - {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", - read_timeout: 10000, - validate: false, - redis_cache: @@redis_client, - cube_options: @@cube_options}) + @@sparql_backends[name][:query] = Goo::SPARQL::Client.new(opts[:query], + protocol: "1.1", + headers: { "Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/sparql-results+json"}, + read_timeout: 10000, + validate: false, + redis_cache: @@redis_client, + logger: query_logging? ? Logger.new(query_logging_file) : nil) + @@sparql_backends[name][:update] = Goo::SPARQL::Client.new(opts[:update], + protocol: "1.1", + headers: { "Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/sparql-results+json"}, + read_timeout: 10000, + validate: false, + redis_cache: @@redis_client) + @@sparql_backends[name][:data] = Goo::SPARQL::Client.new(opts[:data], + protocol: "1.1", + headers: { "Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/sparql-results+json"}, + read_timeout: 10000, + validate: false, + redis_cache: @@redis_client) @@sparql_backends[name][:backend_name] = opts[:backend_name] @@sparql_backends.freeze end - def self.test_reset - if @@sparql_backends[:main][:query].url.to_s["localhost"].nil? - raise Exception, "only for testing" - end - @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/", - {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", - read_timeout: 300, - redis_cache: @@redis_client }) + def self.main_lang + @@main_lang + end + + def self.main_lang=(value) + @@main_lang = value end def self.use_cache=(value) @@ -115,6 +172,25 @@ def self.queries_debug? return @@debug_enabled end + def self.query_logging? + @@query_logging + end + + def self.query_logging_file + @@query_logging_file + end + + def self.query_logging=(value) + @@query_logging = value + end + def self.query_logging_file=(value) + @@query_logging_file = value + end + + def self.logger + return @@sparql_backends[:main][:query].logger + end + def self.add_search_backend(name, *opts) opts = opts[0] unless opts.include? :service @@ -134,6 +210,12 @@ def self.add_redis_backend(*opts) set_sparql_cache end + def self.add_query_logger(enabled: false, file: ) + @@query_logging = enabled + @@query_logging_file = file + set_query_logging + end + def self.set_sparql_cache if @@sparql_backends.length > 0 && @@use_cache @@sparql_backends.each do |k,epr| @@ -150,38 +232,19 @@ def self.set_sparql_cache end end - def self.set_cube_client - if @@sparql_backends.length > 0 && @@cube_options + + def self.set_query_logging + if @@sparql_backends.length > 0 && query_logging? @@sparql_backends.each do |k,epr| - epr[:query].cube_options= @@cube_options - epr[:data].cube_options= @@cube_options - epr[:update].cube_options= @@cube_options + epr[:query].logger = Logger.new(query_logging_file) end - puts "Using cube options in Goo #{@@cube_options}" elsif @@sparql_backends.length > 0 @@sparql_backends.each do |k,epr| - epr[:query].cube_options= nil - epr[:data].cube_options= nil - epr[:update].cube_options=nil + epr[:query].logger = nil end end end - def self.enable_cube - if not block_given? - raise ArgumentError, "Cube configuration needs to receive a code block" - end - cube_options = {} - yield cube_options - @@cube_options = cube_options - set_cube_client - end - - def self.disable_cube - @@cube_options = nil - set_cube_client - end - def self.configure_sanity_check() unless @@namespaces.length > 0 raise ArgumentError, "Namespaces needs to be provided." @@ -196,11 +259,10 @@ def self.configure raise ArgumentError, "Configuration needs to receive a code block" end yield self - configure_sanity_check() + configure_sanity_check - if @@search_backends.length > 0 - @@search_backends.each { |name, val| @@search_connection[name] = RSolr.connect(:url => search_conf(name), :read_timeout => 1800, :open_timeout => 1800) } - end + + init_search_connections @@namespaces.freeze @@sparql_backends.freeze @@ -224,8 +286,44 @@ def self.search_conf(name=:main) return @@search_backends[name][:service] end - def self.search_connection(name=:main) - return @@search_connection[name] + def self.search_connection(collection_name) + return search_client(collection_name).solr + end + + def self.search_client(collection_name) + @@search_connection[collection_name] + end + + def self.add_search_connection(collection_name, search_backend = :main, &block) + @@search_collections[collection_name] = { + search_backend: search_backend, + block: block_given? ? block : nil + } + end + + def self.search_connections + @@search_connection + end + + def self.init_search_connection(collection_name, search_backend = :main, block = nil, force: false) + return @@search_connection[collection_name] if @@search_connection[collection_name] && !force + + @@search_connection[collection_name] = SOLR::SolrConnector.new(search_conf(search_backend), collection_name) + if block + block.call(@@search_connection[collection_name].schema_generator) + @@search_connection[collection_name].enable_custom_schema + end + @@search_connection[collection_name].init(force) + @@search_connection[collection_name] + end + + + def self.init_search_connections(force=false) + @@search_collections.each do |collection_name, backend| + search_backend = backend[:search_backend] + block = backend[:block] + init_search_connection(collection_name, search_backend, block, force: force) + end end def self.sparql_query_client(name=:main) @@ -244,6 +342,10 @@ def self.sparql_backend_name(name=:main) return @@sparql_backends[name][:backend_name] end + def self.portal_language + @@main_languages.first.downcase.to_sym + end + def self.id_prefix return @@id_prefix end @@ -313,4 +415,3 @@ def call(env) Goo::Filter = Goo::Base::Filter Goo::Pattern = Goo::Base::Pattern Goo::Collection = Goo::Base::Collection - diff --git a/lib/goo/base/attribute_proxy.rb b/lib/goo/base/attribute_proxy.rb deleted file mode 100644 index 5600256e..00000000 --- a/lib/goo/base/attribute_proxy.rb +++ /dev/null @@ -1,57 +0,0 @@ - -module Goo - module Base - - class AttributeValueProxy - def initialize(validator,internals) - @validator = validator - @internals = internals - end - - def cardinality_transform(attr, value, current_value) - if @validator.nil? - unless value.kind_of? Array - raise ArgumentError, "Attribute '#{attr} must be an array. No cardinality configured.'" - end - return value - end - if value.kind_of? Array - if @validator.options[:max] and value.length > @validator.options[:max] - #TODO review this - return value[0] if attr == :prefLabel - raise ArgumentError, "Attribute '#{attr}' does not satisfy max cardinality." - end - if @validator.options[:min] and value.length < @validator.options[:min] - raise ArgumentError, "Attribute '#{attr}' does not satisfy min cardinality." - end - if @validator.options[:max] and @validator.options[:max] == 1 - return value[0] - end - else #not an array - if (not @validator.options[:max]) or @validator.options[:max] > 1 - return [value] - end - if @validator.options[:max] and @validator.options[:max] == 1 - return value - end - if @validator.options[:min] and @validator.options[:min] > 0 - return [value] - end - end - if not value.kind_of? Array and current_value.kind_of? Array - raise ArgumentError, - "Multiple value objects cannot be replaced for non-array objects" - end - if value.kind_of? Array then value else [value] end - end - - def call(*args) - options = args[0] - value = options[:value] - attr = options[:attr] - current_value = options[:current_value] - tvalue = cardinality_transform(attr,value,current_value) - end - end - end -end diff --git a/lib/goo/base/filter.rb b/lib/goo/base/filter.rb index 66f2095d..50fa58ec 100644 --- a/lib/goo/base/filter.rb +++ b/lib/goo/base/filter.rb @@ -11,50 +11,54 @@ def initialize(pattern) end def >(value) - @filter_tree << FILTER_TUPLE.new(:>,value) + @filter_tree << FILTER_TUPLE.new(:>, value) self end def <(value) - @filter_tree << FILTER_TUPLE.new(:<,value) + @filter_tree << FILTER_TUPLE.new(:<, value) self end def <=(value) - @filter_tree << FILTER_TUPLE.new(:<=,value) + @filter_tree << FILTER_TUPLE.new(:<=, value) self end def >=(value) - @filter_tree << FILTER_TUPLE.new(:>=,value) + @filter_tree << FILTER_TUPLE.new(:>=, value) self end def or(value) - @filter_tree << FILTER_TUPLE.new(:or,value) + @filter_tree << FILTER_TUPLE.new(:or, value) self end def ==(value) - @filter_tree << FILTER_TUPLE.new(:==,value) + @filter_tree << FILTER_TUPLE.new(:==, value) self end def and(value) - @filter_tree << FILTER_TUPLE.new(:and,value) + @filter_tree << FILTER_TUPLE.new(:and, value) self end def unbound - @filter_tree << FILTER_TUPLE.new(:unbound,nil) + @filter_tree << FILTER_TUPLE.new(:unbound, nil) self end def bound - @filter_tree << FILTER_TUPLE.new(:bound,nil) + @filter_tree << FILTER_TUPLE.new(:bound, nil) self end + def regex(value) + @filter_tree << FILTER_TUPLE.new(:regex, value) + self + end end end end diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb index c12f6203..375ca32d 100644 --- a/lib/goo/base/resource.rb +++ b/lib/goo/base/resource.rb @@ -15,7 +15,7 @@ class Resource attr_reader :modified_attributes attr_reader :errors attr_reader :aggregates - attr_reader :unmapped + attr_writer :unmapped attr_reader :id @@ -42,9 +42,7 @@ def valid? self.class.attributes.each do |attr| inst_value = self.instance_variable_get("@#{attr}") attr_errors = Goo::Validators::Enforce.enforce(self,attr,inst_value) - unless attr_errors.nil? - validation_errors[attr] = attr_errors - end + validation_errors[attr] = attr_errors unless attr_errors.nil? end if !@persistent && validation_errors.length == 0 @@ -70,33 +68,16 @@ def valid? end def id=(new_id) - if !@id.nil? and @persistent - raise ArgumentError, "The id of a persistent object cannot be changed." - end + raise ArgumentError, "The id of a persistent object cannot be changed." if !@id.nil? and @persistent raise ArgumentError, "ID must be an RDF::URI" unless new_id.kind_of?(RDF::URI) @id = new_id end def id - if @id.nil? - if self.class.name_with == :id - raise IDGenerationError, ":id must be set if configured in name_with" - end - custom_name = self.class.name_with - if custom_name.instance_of?(Symbol) - @id = id_from_attribute() - elsif custom_name - begin - @id = custom_name.call(self) - rescue => e - raise IDGenerationError, "Problem with custom id generation: #{e.message}" - end - else - raise IDGenerationError, "custom_name is nil. settings for this model are incorrect." + @id = generate_id if @id.nil? + + @id end - end - return @id - end def persistent? return @persistent @@ -110,22 +91,20 @@ def modified? return modified_attributes.length > 0 end - def exist?(from_valid=false) - #generate id with proc - begin - id() unless self.class.name_with.kind_of?(Symbol) - rescue IDGenerationError - end - - _id = @id - if _id.nil? && !from_valid && self.class.name_with.is_a?(Symbol) + def exist?(from_valid = false) begin - _id = id_from_attribute() + id unless self.class.name_with.kind_of?(Symbol) rescue IDGenerationError + # Ignored end + + _id = @id + if from_valid || _id.nil? + _id = generate_id rescue _id = nil end + return false unless _id - return Goo::SPARQL::Queries.model_exist(self,id=_id) + Goo::SPARQL::Queries.model_exist(self, id = _id) end def fully_loaded? @@ -140,32 +119,40 @@ def missing_load_attributes def unmapped_set(attribute,value) @unmapped ||= {} - (@unmapped[attribute] ||= Set.new) << value + @unmapped[attribute] ||= Set.new + @unmapped[attribute].merge(Array(value)) unless value.nil? + end + + def unmapped_get(attribute) + @unmapped[attribute] end def unmmaped_to_array cpy = {} + @unmapped.each do |attr,v| cpy[attr] = v.to_a end @unmapped = cpy end + def unmapped(*args) + @unmapped&.transform_values do |language_values| + self.class.not_show_all_languages?(language_values, args) ? language_values.values.flatten: language_values + end + end + def delete(*args) if self.kind_of?(Goo::Base::Enum) - unless args[0] && args[0][:init_enum] - raise ArgumentError, "Enums cannot be deleted" - end + raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum] end raise ArgumentError, "This object is not persistent and cannot be deleted" if !@persistent - if !fully_loaded? + unless fully_loaded? missing = missing_load_attributes - options_load = { models: [ self ], klass: self.class, :include => missing } - if self.class.collection_opts - options_load[:collection] = self.collection - end + options_load = { models: [self], klass: self.class, :include => missing } + options_load[:collection] = self.collection if self.class.collection_opts Goo::SPARQL::Queries.model_load(options_load) end @@ -181,26 +168,24 @@ def delete(*args) end @persistent = false @modified = true - if self.class.inmutable? && self.class.inm_instances - self.class.load_inmutable_instances + + if self.class.after_destroy? + self.class.call_after_destroy(self) end + return nil end def bring(*opts) opts.each do |k| if k.kind_of?(Hash) - k.each do |k2,v| - if self.class.handler?(k2) - raise ArgumentError, "Unable to bring a method based attr #{k2}" - end - self.instance_variable_set("@#{k2}",nil) + k.each do |k2,_| + instance_variable_set("@#{k2}", nil) + send(k2) if self.class.handler?(k2) end else - if self.class.handler?(k) - raise ArgumentError, "Unable to bring a method based attr #{k}" - end - self.instance_variable_set("@#{k}",nil) + instance_variable_set("@#{k}", nil) + send(k) if self.class.handler?(k) end end query = self.class.where.models([self]).include(*opts) @@ -214,9 +199,7 @@ def bring(*opts) def graph opts = self.class.collection_opts - if opts.nil? - return self.class.uri_type - end + return self.class.uri_type if opts.nil? col = collection if col.is_a?Array if col.length == 1 @@ -228,79 +211,14 @@ def graph return col ? col.id : nil end - def self.map_attributes(inst,equivalent_predicates=nil) - if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) || - (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?) - raise ArgumentError, "Resource.map_attributes only works for :unmapped instances" - end - klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class - unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped - list_attrs = klass.attributes(:list) - unmapped_string_keys = Hash.new - unmapped.each do |k,v| - unmapped_string_keys[k.to_s] = v - end - klass.attributes.each do |attr| - next if inst.class.collection?(attr) #collection is already there - next unless inst.respond_to?(attr) - attr_uri = klass.attribute_uri(attr,inst.collection).to_s - if unmapped_string_keys.include?(attr_uri.to_s) || - (equivalent_predicates && equivalent_predicates.include?(attr_uri)) - object = nil - if !unmapped_string_keys.include?(attr_uri) - equivalent_predicates[attr_uri].each do |eq_attr| - if object.nil? and !unmapped_string_keys[eq_attr].nil? - object = unmapped_string_keys[eq_attr].dup - else - if object.is_a?Array - if !unmapped_string_keys[eq_attr].nil? - object.concat(unmapped_string_keys[eq_attr]) - end - end - end - end - if object.nil? - inst.send("#{attr}=", - list_attrs.include?(attr) ? [] : nil, on_load: true) - next - end - else - object = unmapped_string_keys[attr_uri] - end - object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object } - if klass.range(attr) - object = object.map { |o| - o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o } - end - unless list_attrs.include?(attr) - object = object.first - end - if inst.respond_to?(:klass) - inst[attr] = object - else - inst.send("#{attr}=",object, on_load: true) - end - else - inst.send("#{attr}=", - list_attrs.include?(attr) ? [] : nil, on_load: true) - if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415" - if attr == :definition - # binding.pry - end - end - end - end - end def collection opts = self.class.collection_opts if opts.instance_of?(Symbol) if self.class.attributes.include?(opts) value = self.send("#{opts}") - if value.nil? - raise ArgumentError, "Collection `#{opts}` is nil" - end + raise ArgumentError, "Collection `#{opts}` is nil" if value.nil? return value else raise ArgumentError, "Collection `#{opts}` is not an attribute" @@ -315,26 +233,46 @@ def add_aggregate(attribute,aggregate,value) def save(*opts) if self.kind_of?(Goo::Base::Enum) - unless opts[0] && opts[0][:init_enum] - raise ArgumentError, "Enums can only be created on initialization" - end + raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum] end batch_file = nil - if opts && opts.length > 0 - if opts.first.is_a?(Hash) && opts.first[:batch] && opts.first[:batch].is_a?(File) + callbacks = true + if opts && opts.length > 0 && opts.first.is_a?(Hash) + if opts.first[:batch] && opts.first[:batch].is_a?(File) batch_file = opts.first[:batch] end + + callbacks = opts.first[:callbacks] end if !batch_file - if not modified? - return self + return self if !modified? && persistent? + raise Goo::Base::NotValidException, "Object is not valid. Errors: #{errors}" unless valid? + end + + #set default values before saving + unless self.persistent? + self.class.attributes_with_defaults.each do |attr| + value = self.send("#{attr}") + if value.nil? + value = self.class.default(attr) + value = value.call(self) if value.is_a?(Proc) + self.send("#{attr}=", value) + end + end + end + + #call update callback before saving + if callbacks + self.class.attributes_with_callbacks.each do |attr| + Goo::Validators::Enforce.enforce_callbacks(self, attr) end - raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid? end graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self) - graph = self.graph() + graph = self.graph + + if graph_delete and graph_delete.size > 0 begin Goo.sparql_update_client.delete_data(graph_delete, graph: graph) @@ -356,7 +294,7 @@ def save(*opts) batch_file.write(lines.join("")) batch_file.flush() else - Goo.sparql_update_client.insert_data(graph_insert, graph: graph) + Goo.sparql_update_client.insert_data(graph_insert, graph: graph, use_insert_data: !Goo.backend_vo?) end rescue Exception => e raise e @@ -368,9 +306,11 @@ def save(*opts) @modified_attributes = Set.new @persistent = true - if self.class.inmutable? && self.class.inm_instances - self.class.load_inmutable_instances + + if self.class.after_save? + self.class.call_after_save(self) end + return self end @@ -408,9 +348,7 @@ def to_hash end end @unmapped.each do |attr,values| - unless all_attr_uris.include?(attr) - attr_hash[attr] = values.map { |v| v.to_s } - end + attr_hash[attr] = values.map { |v| v.to_s } unless all_attr_uris.include?(attr) end end attr_hash[:id] = @id @@ -430,18 +368,73 @@ def self.range_object(attr,id) return range_object end - def self.find(id, *options) - if !id.instance_of?(RDF::URI) && self.name_with == :id - id = RDF::URI.new(id) + + + def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false) + if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) || + (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?) + raise ArgumentError, "Resource.map_attributes only works for :unmapped instances" end - unless id.instance_of?(RDF::URI) - id = id_from_unique_attribute(name_with(),id) + klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class + unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped(include_languages: include_languages) + list_attrs = klass.attributes(:list) + unmapped_string_keys = Hash.new + unmapped.each do |k,v| + unmapped_string_keys[k.to_s] = v end - if self.inmutable? && self.inm_instances && self.inm_instances[id] - w = Goo::Base::Where.new(self) - w.instance_variable_set("@result", [self.inm_instances[id]]) - return w + klass.attributes.each do |attr| + next if inst.class.collection?(attr) #collection is already there + next unless inst.respond_to?(attr) + attr_uri = klass.attribute_uri(attr,inst.collection).to_s + if unmapped_string_keys.include?(attr_uri.to_s) || equivalent_predicates&.include?(attr_uri) + object = nil + + if unmapped_string_keys.include?(attr_uri) + object = unmapped_string_keys[attr_uri] + else + equivalent_predicates[attr_uri].each do |eq_attr| + next if unmapped_string_keys[eq_attr].nil? + + if object.nil? + object = unmapped_string_keys[eq_attr].dup + elsif object.is_a?(Array) + object.concat(unmapped_string_keys[eq_attr]) + end + end + + if object.nil? + inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true) + next + end + end + + if object.is_a?(Hash) + object = object.transform_values{|values| Array(values).map{|o|o.is_a?(RDF::URI) ? o : o.object}} + else + object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object} + end + + if klass.range(attr) + object = object.map { |o| + o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o } + end + + object = object.first unless list_attrs.include?(attr) || include_languages + if inst.respond_to?(:klass) + inst[attr] = object + else + inst.send("#{attr}=",object, on_load: true) + end + else + inst.send("#{attr}=", + list_attrs.include?(attr) ? [] : nil, on_load: true) + end + end + end + def self.find(id, *options) + id = RDF::URI.new(id) if !id.instance_of?(RDF::URI) && self.name_with == :id + id = id_from_unique_attribute(name_with(),id) unless id.instance_of?(RDF::URI) options_load = { ids: [id], klass: self }.merge(options[-1] || {}) options_load[:find] = true where = Goo::Base::Where.new(self) @@ -462,10 +455,30 @@ def self.all end protected + def id_from_attribute() uattr = self.class.name_with uvalue = self.send("#{uattr}") - return self.class.id_from_unique_attribute(uattr,uvalue) + return self.class.id_from_unique_attribute(uattr, uvalue) + end + + def generate_id + return nil unless self.class.name_with + + raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id + custom_name = self.class.name_with + if custom_name.instance_of?(Symbol) + id = id_from_attribute + elsif custom_name + begin + id = custom_name.call(self) + rescue => e + raise IDGenerationError, "Problem with custom id generation: #{e.message}" + end + else + raise IDGenerationError, "custom_name is nil. settings for this model are incorrect." + end + id end end diff --git a/lib/goo/base/settings/attribute.rb b/lib/goo/base/settings/attribute.rb new file mode 100644 index 00000000..ff925e4d --- /dev/null +++ b/lib/goo/base/settings/attribute.rb @@ -0,0 +1,240 @@ +module Goo + module Base + module Settings + module AttributeSettings + + def attribute(*args) + options = args.reverse + attr_name = options.pop + attr_name = attr_name.to_sym + options = options.pop + options = {} if options.nil? + + options[:enforce] ||= [] + + set_data_type(options) + set_no_list_by_default(options) + + @model_settings[:attributes][attr_name] = options + load_yaml_scheme_options(attr_name) + shape_attribute(attr_name) + namespace = attribute_namespace(attr_name) || @model_settings[:namespace] + vocab = Goo.vocabulary(namespace) + if options[:property].is_a?(Proc) + @attribute_uris[attr_name] = options[:property] + else + @attribute_uris[attr_name] = vocab[options[:property] || attr_name] + end + if options[:enforce].include?(:unique) && options[:enforce].include?(:list) + raise ArgumentError, ":list options cannot be combined with :list" + end + set_range(attr_name) + end + + def shape_attribute(attr) + return if attr == :resource_id + + attr = attr.to_sym + define_method("#{attr}=") do |*args| + if self.class.handler?(attr) + raise ArgumentError, "Method based attributes cannot be set" + end + if self.class.inverse?(attr) && !(args && args.last.instance_of?(Hash) && args.last[:on_load]) + raise ArgumentError, "`#{attr}` is an inverse attribute. Values cannot be assigned." + end + @loaded_attributes.add(attr) + value = args[0] + unless args.last.instance_of?(Hash) and args.last[:on_load] + if self.persistent? and self.class.name_with == attr + raise ArgumentError, "`#{attr}` attribute is used to name this resource and cannot be modified." + end + prev = self.instance_variable_get("@#{attr}") + if !prev.nil? and !@modified_attributes.include?(attr) + if prev != value + @previous_values = @previous_values || {} + @previous_values[attr] = prev + end + end + @modified_attributes.add(attr) + end + if value.instance_of?(Array) + value = value.dup.freeze + end + self.instance_variable_set("@#{attr}", value) + end + define_method("#{attr}") do |*args| + attr_value = self.instance_variable_get("@#{attr}") + + if self.class.not_show_all_languages?(attr_value, args) + is_array = attr_value.values.first.is_a?(Array) + attr_value = attr_value.values.flatten + attr_value = attr_value.first unless is_array + end + + if self.class.handler?(attr) + if @loaded_attributes.include?(attr) + return attr_value + end + value = self.send("#{self.class.handler(attr)}") + self.instance_variable_set("@#{attr}", value) + @loaded_attributes << attr + return value + end + + if (not @persistent) or @loaded_attributes.include?(attr) + return attr_value + else + # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission) + raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}." + end + end + end + + def attributes(*options) + if options and options.length > 0 + option = options.first + + if option == :all + return @model_settings[:attributes].keys + end + + if option == :inverse + return @model_settings[:attributes].select { |_, v| v[:inverse] }.keys + end + + attrs = @model_settings[:attributes].select { |_, opts| opts[:enforce].include?(option) }.keys + + attrs.concat(attributes(:inverse)) if option == :list + + return attrs + end + + @model_settings[:attributes].select { |k, attr| attr[:inverse].nil? && !handler?(k) }.keys + + end + + def attributes_with_defaults + @model_settings[:attributes].select { |_, opts| opts[:default] }.keys + end + + def attribute_namespace(attr) + attribute_settings(attr)[:namespace] + end + + def default(attr) + attribute_settings(attr)[:default] + end + + def range(attr) + @model_settings[:range][attr] + end + + def attribute_settings(attr) + @model_settings[:attributes][attr] + end + + def required?(attr) + return false if attribute_settings(attr).nil? + attribute_settings(attr)[:enforce].include?(:existence) + end + + def unique?(attr) + return false if attribute_settings(attr).nil? + attribute_settings(attr)[:enforce].include?(:unique) + end + + def datatype(attr) + enforced = attribute_settings(attr)[:enforce].dup + return :string if enforced.nil? + + enforced.delete(:list) + enforced.delete(:no_list) + + enforced.find { |e| Goo::Validators::DataType.ids.include?(e) } || :string + end + + def list?(attr) + return false if attribute_settings(attr).nil? + attribute_settings(attr)[:enforce].include?(:list) + end + + def transitive?(attr) + return false unless @model_settings[:attributes].include?(attr) + attribute_settings(attr)[:transitive] == true + end + + def alias?(attr) + return false unless @model_settings[:attributes].include?(attr) + attribute_settings(attr)[:alias] == true + end + + def handler?(attr) + return false if attribute_settings(attr).nil? + !attribute_settings(attr)[:handler].nil? + end + + def handler(attr) + return false if attribute_settings(attr).nil? + attribute_settings(attr)[:handler] + end + + def inverse?(attr) + return false if attribute_settings(attr).nil? + !attribute_settings(attr)[:inverse].nil? + end + + def inverse_opts(attr) + attribute_settings(attr)[:inverse] + end + + def attribute_uri(attr, *args) + attr = attr.to_sym + if attr == :id + raise ArgumentError, ":id cannot be treated as predicate for .where, use find " + end + uri = @attribute_uris[attr] + if uri.is_a?(Proc) + uri = uri.call(*args.flatten) + end + return uri unless uri.nil? + attr_string = attr.to_s + Goo.namespaces.keys.each do |ns| + nss = ns.to_s + if attr_string.start_with?(nss) + return Goo.vocabulary(ns)[attr_string[nss.length + 1..-1]] + end + end + + Goo.vocabulary(nil)[attr] + end + + + def indexable?(attr) + setting = attribute_settings(attr.to_sym) + setting && (setting[:index].nil? || setting[:index] == true) + end + + def fuzzy_searchable?(attr) + attribute_settings(attr)[:fuzzy_search] == true + end + + + private + + def set_no_list_by_default(options) + if options[:enforce].nil? or !options[:enforce].include?(:list) + options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list] + end + end + + def set_data_type(options) + if options[:type] + options[:enforce] += Array(options[:type]) + options[:enforce].uniq! + options.delete :type + end + end + end + end + end +end diff --git a/lib/goo/base/settings/hooks.rb b/lib/goo/base/settings/hooks.rb new file mode 100644 index 00000000..7925b2a0 --- /dev/null +++ b/lib/goo/base/settings/hooks.rb @@ -0,0 +1,62 @@ +require 'yaml' +require_relative '../../utils/callbacks_utils' + +module Goo + module Base + module Settings + module Hooks + + include CallbackRunner + + def after_save(*methods) + @model_settings[:after_save] ||= [] + @model_settings[:after_save].push(*methods) + end + + def after_destroy(*methods) + @model_settings[:after_destroy] ||= [] + @model_settings[:after_destroy].push(*methods) + end + + def after_save_callbacks + Array(@model_settings[:after_save]) + end + + def after_destroy_callbacks + Array(@model_settings[:after_destroy]) + end + + def after_save? + !after_save_callbacks.empty? + end + + def after_destroy? + !after_destroy_callbacks.empty? + end + + def call_after_save(inst) + run_callbacks(inst, after_save_callbacks) + end + + def call_after_destroy(inst) + run_callbacks(inst, after_destroy_callbacks) + end + + def attributes_with_callbacks + (@model_settings[:attributes]. + select{ |attr,opts| opts[:onUpdate] }).keys + end + + + def attribute_callbacks(attr) + @model_settings[:attributes][attr][:onUpdate] + end + + end + end + end +end + + + + diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb index 2a274454..bf2c38da 100644 --- a/lib/goo/base/settings/settings.rb +++ b/lib/goo/base/settings/settings.rb @@ -1,4 +1,7 @@ require 'active_support/core_ext/string' +require_relative 'yaml_settings' +require_relative 'hooks' +require_relative 'attribute' module Goo module Base @@ -11,9 +14,12 @@ module ClassMethods attr_accessor :model_settings attr_reader :model_name attr_reader :attribute_uris + attr_reader :namespace + + include YAMLScheme ,AttributeSettings, Hooks def default_model_options - return {} + {name_with: lambda {|x| uuid_uri_generator(x)}} end def model(*args) @@ -25,19 +31,18 @@ def model(*args) model_name = args[0] @model_name = model_name.to_sym - #a hash with options is expected + # a hash with options is expected options = args.last - @inmutable = (args.include? :inmutable) - if @inmutable - @inm_instances = nil - end @model_settings = default_model_options.merge(options || {}) - unless options.include?:name_with + init_yaml_scheme_settings + + unless options.include? :name_with raise ArgumentError, "The model `#{model_name}` definition should include the :name_with option" end - Goo.add_model(@model_name,self) + + Goo.add_model(@model_name, self) @attribute_uris = {} @namespace = Goo.vocabulary(@model_settings[:namespace]) @uri_type = @namespace[@model_name.to_s.camelize] @@ -45,8 +50,8 @@ def model(*args) @model_settings[:attributes] = {} @model_settings[:rdf_type] = options[:rdf_type] - #registering a new models forces to redo ranges - Goo.models.each do |k,m| + # registering a new models forces to redo ranges + Goo.models.each do |k, m| m.attributes(:all).each do |attr| next if m.range(attr) m.set_range(attr) @@ -54,224 +59,29 @@ def model(*args) end end - def attributes(*options) - if options and options.length > 0 - filt = options.first - if filt == :all - return @model_settings[:attributes].keys - end - if filt == :inverse - return @model_settings[:attributes].keys. - select{ |k| @model_settings[:attributes][k][:inverse] } - end - atts = (@model_settings[:attributes]. - select{ |attr,opts| opts[:enforce].include?(filt) }).keys() - atts.concat(attributes(:inverse)) if filt == :list - return atts - end - return @model_settings[:attributes].keys. - select{ |k| @model_settings[:attributes][k][:inverse].nil? }. - select{ |k| !handler?(k) } - end - - def inmutable? - return @inmutable - end - - def collection?(attr) - return @model_settings[:collection] == attr - end - - def collection_opts - return @model_settings[:collection] - end - - def attributes_with_defaults - return (@model_settings[:attributes]. - select{ |attr,opts| opts[:default] }).keys() - end - - def default(attr) - return @model_settings[:attributes][attr][:default] - end - - def attribute_namespace(attr) - return @model_settings[:attributes][attr][:namespace] - end - - def range(attr) - @model_settings[:range][attr] - end - - def attribute_settings(attr) - @model_settings[:attributes][attr] - end - - def cardinality(attr) - return nil if @model_settings[:attributes][attr].nil? - cardinality = {} - enforce = @model_settings[:attributes][attr][:enforce] - min = enforce.map {|e| e.to_s.split("_").last.to_i if e.to_s.start_with?("min_") }.compact - max = enforce.map {|e| e.to_s.split("_").last.to_i if e.to_s.start_with?("max_") }.compact - cardinality[:min] = min.first unless min.empty? - cardinality[:max] = max.first unless max.empty? - cardinality.empty? ? nil : cardinality - end - - def required?(attr) - return false if @model_settings[:attributes][attr].nil? - @model_settings[:attributes][attr][:enforce].include?(:existence) - end - - def unique?(attr) - return false if @model_settings[:attributes][attr].nil? - @model_settings[:attributes][attr][:enforce].include?(:unique) - end - - def list?(attr) - return false if @model_settings[:attributes][attr].nil? - @model_settings[:attributes][attr][:enforce].include?(:list) - end - - def transitive?(attr) - return false if !@model_settings[:attributes].include?(attr) - return (@model_settings[:attributes][attr][:transitive] == true) - end - - def alias?(attr) - return false if !@model_settings[:attributes].include?(attr) - return (@model_settings[:attributes][attr][:alias] == true) - end - - def handler?(attr) - return false if @model_settings[:attributes][attr].nil? - return (!@model_settings[:attributes][attr][:handler].nil?) - end - - def handler(attr) - return false if @model_settings[:attributes][attr].nil? - return @model_settings[:attributes][attr][:handler] - end - - def inverse?(attr) - return false if @model_settings[:attributes][attr].nil? - return (!@model_settings[:attributes][attr][:inverse].nil?) - end - - def inverse_opts(attr) - return @model_settings[:attributes][attr][:inverse] - end - def set_range(attr) - @model_settings[:attributes][attr][:enforce].each do |opt| + attribute_settings(attr)[:enforce].each do |opt| if Goo.models.include?(opt) || opt.respond_to?(:model_name) || (opt.respond_to?(:new) && opt.new.kind_of?(Struct)) opt = Goo.models[opt] if opt.instance_of?(Symbol) - @model_settings[:range][attr]=opt + @model_settings[:range][attr] = opt break end end - if @model_settings[:attributes][attr][:inverse] - on = @model_settings[:attributes][attr][:inverse][:on] + if attribute_settings(attr)[:inverse] + on = attribute_settings(attr)[:inverse][:on] if Goo.models.include?(on) || on.respond_to?(:model_name) on = Goo.models[on] if on.instance_of?(Symbol) - @model_settings[:range][attr]=on + @model_settings[:range][attr] = on end end end - def attribute(*args) - options = args.reverse - attr_name = options.pop - attr_name = attr_name.to_sym - options = options.pop - options = {} if options.nil? - if options[:enforce].nil? or !options[:enforce].include?(:list) - options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list] - end - @model_settings[:attributes][attr_name] = options - shape_attribute(attr_name) - namespace = attribute_namespace(attr_name) - namespace = namespace || @model_settings[:namespace] - vocab = Goo.vocabulary(namespace) #returns default for nil input - if options[:property].is_a?(Proc) - @attribute_uris[attr_name] = options[:property] - else - @attribute_uris[attr_name] = vocab[options[:property] || attr_name] - end - if options[:enforce].include?(:unique) and options[:enforce].include?(:list) - raise ArgumentError, ":list options cannot be combined with :list" - end - set_range(attr_name) - end - - def attribute_uri(attr,*args) - if attr == :id - raise ArgumentError, ":id cannot be treated as predicate for .where, use find " - end - uri = @attribute_uris[attr] - if uri.is_a?(Proc) - uri = uri.call(*args.flatten) - end - return uri unless uri.nil? - attr_string = attr.to_s - Goo.namespaces.keys.each do |ns| - nss = ns.to_s - if attr_string.start_with?(nss) - return Goo.vocabulary(ns)[attr_string[nss.length+1..-1]] - end - end - #default - return Goo.vocabulary(nil)[attr] + def collection?(attr) + @model_settings[:collection] == attr end - def shape_attribute(attr) - return if attr == :resource_id - attr = attr.to_sym - define_method("#{attr}=") do |*args| - if self.class.handler?(attr) - raise ArgumentError, "Method based attributes cannot be set" - end - if self.class.inverse?(attr) && !(args && args.last.instance_of?(Hash) && args.last[:on_load]) - raise ArgumentError, - "`#{attr}` is an inverse attribute. Values cannot be assigned." - end - @loaded_attributes.add(attr) - value = args[0] - unless args.last.instance_of?(Hash) and args.last[:on_load] - if self.persistent? and self.class.name_with == attr - raise ArgumentError, - "`#{attr}` attribute is used to name this resource and cannot be modified." - end - prev = self.instance_variable_get("@#{attr}") - if !prev.nil? and !@modified_attributes.include?(attr) - if prev != value - @previous_values = @previous_values || {} - @previous_values[attr] = prev - end - end - @modified_attributes.add(attr) - end - if value.instance_of?(Array) - value = value.dup.freeze - end - self.instance_variable_set("@#{attr}",value) - end - define_method("#{attr}") do |*args| - if self.class.handler?(attr) - if @loaded_attributes.include?(attr) - return self.instance_variable_get("@#{attr}") - end - value = self.send("#{self.class.handler(attr)}") - self.instance_variable_set("@#{attr}",value) - @loaded_attributes << attr - return value - end - if (not @persistent) or @loaded_attributes.include?(attr) - return self.instance_variable_get("@#{attr}") - else - raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}." - end - end + def collection_opts + @model_settings[:collection] end def uuid_uri_generator(inst) @@ -280,19 +90,14 @@ def uuid_uri_generator(inst) if Goo.id_prefix return RDF::URI.new(Goo.id_prefix + model_name_uri + '/' + Goo.uuid) end - return namespace[ model_name_uri + '/' + Goo.uuid] + namespace[model_name_uri + '/' + Goo.uuid] end def uri_type(*args) - if @model_settings[:rdf_type] - return @model_settings[:rdf_type].call(*args) - end - return @uri_type + @model_settings[:rdf_type] ? @model_settings[:rdf_type].call(*args) : @uri_type end + alias :type_uri :uri_type - def namespace - return @namespace - end def id_prefix model_name_uri = model_name.to_s @@ -300,16 +105,15 @@ def id_prefix if Goo.id_prefix return RDF::URI.new(Goo.id_prefix + model_name_uri + '/') end - return namespace[model_name_uri + '/'] + namespace[model_name_uri + '/'] end - def id_from_unique_attribute(attr,value_attr) + def id_from_unique_attribute(attr, value_attr) if value_attr.nil? raise Goo::Base::IDGenerationError, "`#{attr}` value is nil. Id for resource cannot be generated." end uri_last_fragment = CGI.escape(value_attr) - model_prefix_uri = id_prefix() - return model_prefix_uri + uri_last_fragment + id_prefix + uri_last_fragment end def enum(*values) @@ -320,25 +124,11 @@ def enum(*values) end def name_with - return @model_settings[:name_with] - end - - def load_inmutable_instances - #TODO this should be SYNC - @inm_instances = nil - ins = self.where.include(self.attributes).all - @inm_instances = {} - ins.each do |ins| - @inm_instances[ins.id] = ins - end + @model_settings[:name_with] end def attribute_loaded?(attr) - return @loaded_attributes.include?(attr) - end - - def inm_instances - @inm_instances + @loaded_attributes.include?(attr) end def struct_object(attrs) @@ -349,12 +139,12 @@ def struct_object(attrs) attrs << :unmapped attrs << collection_opts if collection_opts attrs.uniq! - return Struct.new(*attrs) + Struct.new(*attrs) end STRUCT_CACHE = {} ## - # Return a struct-based, + # Return a struct-based, # read-only instance for a class that is populated with the contents of `attributes` def read_only(attributes) if !attributes.is_a?(Hash) || attributes.empty? @@ -368,10 +158,18 @@ def read_only(attributes) cls = STRUCT_CACHE[attributes.keys.hash] instance = cls.new instance.klass = self - attributes.each {|k,v| instance[k] = v} + attributes.each { |k, v| instance[k] = v } instance end + def show_all_languages?(args) + args.first.is_a?(Hash) && args.first.keys.include?(:include_languages) && args.first[:include_languages] + end + + def not_show_all_languages?(values, args) + values.is_a?(Hash) && !show_all_languages?(args) + end + end end end diff --git a/lib/goo/base/settings/yaml_settings.rb b/lib/goo/base/settings/yaml_settings.rb new file mode 100644 index 00000000..8a931b3a --- /dev/null +++ b/lib/goo/base/settings/yaml_settings.rb @@ -0,0 +1,45 @@ +require 'yaml' + +module Goo + module Base + module Settings + module YAMLScheme + attr_reader :yaml_settings + + def init_yaml_scheme_settings + scheme_file_path = @model_settings[:scheme] + @yaml_settings = read_yaml_settings_file(scheme_file_path) + end + + def attribute_yaml_settings(attr) + + return {} if yaml_settings.nil? + + yaml_settings[attr.to_sym] + end + + + + private + + def load_yaml_scheme_options(attr) + settings = attribute_settings(attr) + yaml_settings = attribute_yaml_settings(attr) + settings.merge! yaml_settings unless yaml_settings.nil? || yaml_settings.empty? + end + + def read_yaml_settings_file(scheme_file_path) + return if scheme_file_path.nil? + + yaml_contents = File.read(scheme_file_path) rescue return + + YAML.safe_load(yaml_contents, symbolize_names: true) + end + end + end + end +end + + + + diff --git a/lib/goo/base/where.rb b/lib/goo/base/where.rb index 5bc0fa8c..8380424a 100644 --- a/lib/goo/base/where.rb +++ b/lib/goo/base/where.rb @@ -6,6 +6,7 @@ class Where AGGREGATE_PATTERN = Struct.new(:pattern,:aggregate) attr_accessor :where_options_load + include Goo::SPARQL::Processor def initialize(klass,*match_patterns) if Goo.queries_debug? && Thread.current[:ncbo_debug].nil? @@ -122,113 +123,7 @@ def unmmaped_predicates() end def process_query(count=false) - if Goo.queries_debug? && Thread.current[:ncbo_debug] - tstart = Time.now - query_resp = process_query_intl(count=count) - (Thread.current[:ncbo_debug][:goo_process_query] ||= []) << (Time.now - tstart) - return query_resp - end - return process_query_intl(count=count) - end - - def process_query_intl(count=false) - if @models == [] - @result = [] - return @result - end - - @include << @include_embed if @include_embed.length > 0 - - @predicates = unmmaped_predicates() - @equivalent_predicates = retrieve_equivalent_predicates() - - options_load = { models: @models, include: @include, ids: @ids, - graph_match: @pattern, klass: @klass, - filters: @filters, order_by: @order_by , - read_only: @read_only, rules: @rules, - predicates: @predicates, - no_graphs: @no_graphs, - equivalent_predicates: @equivalent_predicates } - - options_load.merge!(@where_options_load) if @where_options_load - if !@klass.collection_opts.nil? and !options_load.include?(:collection) - raise ArgumentError, "Collection needed call `#{@klass.name}`" - end - - ids = nil - if @index_key - raise ArgumentError, "Redis is not configured" unless Goo.redis_client - rclient = Goo.redis_client - cache_key = cache_key_for_index(@index_key) - raise ArgumentError, "Index not found" unless rclient.exists(cache_key) - if @page_i - if !@count - @count = rclient.llen(cache_key) - end - rstart = (@page_i -1) * @page_size - rstop = (rstart + @page_size) -1 - ids = rclient.lrange(cache_key,rstart,rstop) - else - ids = rclient.lrange(cache_key,0,-1) - end - ids = ids.map { |i| RDF::URI.new(i) } - end - - if @page_i && !@index_key - page_options = options_load.dup - page_options.delete(:include) - page_options[:include_pagination] = @include - if not @pre_count.nil? - @count = @pre_count - else - if !@count && @do_count - page_options[:count] = :count - @count = Goo::SPARQL::Queries.model_load(page_options).to_i - end - end - page_options.delete :count - page_options[:query_options] = @query_options - page_options[:page] = { page_i: @page_i, page_size: @page_size } - models_by_id = Goo::SPARQL::Queries.model_load(page_options) - options_load[:models] = models_by_id.values - - #models give the constraint - options_load.delete :graph_match - elsif count - count_options = options_load.dup - count_options.delete(:include) - count_options[:count] = :count - return Goo::SPARQL::Queries.model_load(count_options).to_i - end - - if @indexing - #do not care about include values - @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values) - return @result - end - - options_load[:ids] = ids if ids - models_by_id = {} - if (@page_i && options_load[:models].length > 0) || - (!@page_i && (@count.nil? || @count > 0)) - models_by_id = Goo::SPARQL::Queries.model_load(options_load) - if @aggregate - if models_by_id.length > 0 - options_load_agg = { models: models_by_id.values, klass: @klass, - filters: @filters, read_only: @read_only, - aggregate: @aggregate, rules: @rules } - - options_load_agg.merge!(@where_options_load) if @where_options_load - Goo::SPARQL::Queries.model_load(options_load_agg) - end - end - end - unless @page_i - @result = @models ? @models : models_by_id.values - else - @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values) - end - @result + process_query_call(count = count) end def disable_rules @@ -273,13 +168,26 @@ def index_as(index_key,max=nil) return rclient.llen(final_key) end - def all - if @result.nil? && @klass.inmutable? && @klass.inm_instances - if @pattern.nil? && @filters.nil? - @result = @klass.inm_instances.values - end + def paginated_all(page_size=1000) + page = 1 + page_size = 10000 + result = [] + old_count = -1 + count = 0 + while count != old_count + old_count = count + @page_i = page + @page_size = page_size + result += process_query(count=false) + page += 1 + count = result.length end - process_query unless @result + result + end + + def all + return @result if @result + process_query @result end alias_method :to_a, :all @@ -352,13 +260,13 @@ def include(*options) options.each do |opt| if opt.instance_of?(Symbol) if @klass.handler?(opt) - raise ArgumentError, "Method based attribute cannot be included" + next end end if opt.instance_of?(Hash) opt.each do |k,v| if @klass.handler?(k) - raise ArgumentError, "Method based attribute cannot be included" + next end end end diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb new file mode 100644 index 00000000..37b59e01 --- /dev/null +++ b/lib/goo/config/config.rb @@ -0,0 +1,72 @@ +require 'ostruct' + +module Goo + extend self + attr_reader :settings + + @settings = OpenStruct.new + @settings_run = false + + def config(&block) + return if @settings_run + @settings_run = true + + yield @settings if block_given? + + # Set defaults + @settings.goo_backend_name ||= ENV['GOO_BACKEND_NAME'] || '4store' + @settings.goo_port ||= ENV['GOO_PORT'] || 9000 + @settings.goo_host ||= ENV['GOO_HOST'] || 'localhost' + @settings.goo_path_query ||= ENV['GOO_PATH_QUERY'] || '/sparql/' + @settings.goo_path_data ||= ENV['GOO_PATH_DATA'] || '/data/' + @settings.goo_path_update ||= ENV['GOO_PATH_UPDATE'] || '/update/' + @settings.search_server_url ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr' + @settings.goo_redis_host ||= ENV['REDIS_HOST'] || 'localhost' + @settings.goo_redis_port ||= ENV['REDIS_PORT'] || 6379 + @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/' + @settings.query_logging ||= ENV['QUERIES_LOGGING'] || false + @settings.query_logging_file ||= ENV['QUERIES_LOGGING_FILE'] || './sparql.log' + @settings.queries_debug ||= ENV['QUERIES_DEBUG'] || false + @settings.slice_loading_size ||= ENV['GOO_SLICES']&.to_i || 500 + puts "(GOO) >> Using RDF store (#{@settings.goo_backend_name}) #{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}" + puts "(GOO) >> Using term search server at #{@settings.search_server_url}" + puts "(GOO) >> Using Redis instance at #{@settings.goo_redis_host}:#{@settings.goo_redis_port}" + puts "(GOO) >> Using Query logging: #{@settings.query_logging_file}" if @settings.query_logging + + connect_goo + end + + def connect_goo + begin + Goo.configure do |conf| + conf.queries_debug(@settings.queries_debug) + conf.add_sparql_backend(:main, + backend_name: @settings.goo_backend_name, + query: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}", + data: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_data}", + update: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_update}", + options: { rules: :NONE}) + conf.add_search_backend(:main, service: @settings.search_server_url) + conf.add_redis_backend(host: @settings.goo_redis_host, port: @settings.goo_redis_port) + conf.add_query_logger(enabled: @settings.query_logging, file: @settings.query_logging_file) + + conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/")) + conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#")) + conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#")) + conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#")) + conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"), default = true) + conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/")) + conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/")) + conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#")) + conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#")) + conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#")) + conf.add_namespace(:bioportal, RDF::Vocabulary.new(@settings.bioportal_namespace)) + conf.use_cache = false + conf.slice_loading_size = @settings.slice_loading_size + end + rescue StandardError => e + abort("EXITING: Goo cannot connect to triplestore and/or search server:\n #{e}\n#{e.backtrace.join("\n")}") + end + end + +end diff --git a/lib/goo/mixins/sparql_client.rb b/lib/goo/mixins/sparql_client.rb index fcfb7888..587942ab 100644 --- a/lib/goo/mixins/sparql_client.rb +++ b/lib/goo/mixins/sparql_client.rb @@ -6,91 +6,54 @@ def to_uri module RDF def self.URI(*args, &block) - return args.first - end - - class Writer - def validate? - false - end - end - - class Literal - def to_base - text = [] - text << %("#{escape(value)}") - text << "@#{language}" if has_language? - if has_datatype? - if datatype.respond_to?:to_base - text << "^^#{datatype.to_base}" - else - text << "^^<#{datatype.to_s}>" - end - end - text.join "" - end + return RDF::URI.new(*args) end class URI - def initialize(uri_or_options) - case uri_or_options - when Hash - @uri = Addressable::URI.new(uri_or_options) - when Addressable::URI - @uri = uri_or_options - else - @uri = uri_or_options.to_s - #@uri = Addressable::URI.parse(uri_or_options.to_s) - end - rescue Addressable::URI::InvalidURIError => e - raise ArgumentError, e.message - end - - def method_missing(symbol, *args, &block) - unless @uri.respond_to?(symbol) - if (Addressable::URI.instance_methods.include?(symbol) && @uri.instance_of?(String)) - @uri = Addressable::URI.parse(@uri) - end - end - if @uri.respond_to?(symbol) - case result = @uri.send(symbol, *args, &block) - when Addressable::URI - self.class.new(result) - else result - end + # Delegate any undefined method calls to the String object + def method_missing(method, *args, &block) + if self.to_s.respond_to?(method) + self.to_s.send(method, *args, &block) else super end end - def last_part - f = fragment - return f if f - return to_s.split("/")[-1] + # Ensure respond_to? reflects the delegated methods + def respond_to_missing?(method, include_private = false) + self.to_s.respond_to?(method) || super end - def respond_to?(symbol,include_private = false) - @uri.respond_to?(symbol,include_private=false) || super - end + end - def hash - @uri.to_s.hash + class Writer + def validate? + false end - - end #end URI + end class Literal - @@subclasses_by_uri = {} - def self.datatyped_class(uri) - return nil if uri.nil? - if @@subclasses.length != (@@subclasses_by_uri.length + 1) - @@subclasses.each do |child| - if child.const_defined?(:DATATYPE) - @@subclasses_by_uri[child.const_get(:DATATYPE).to_s] = child - end - end - end - return @@subclasses_by_uri[uri] - end + class DateTime < Temporal + # Override RDF's default DateTime format to match what 4store accepts. + # Remove first to avoid Ruby's "already initialized constant" warning. + remove_const(:FORMAT) if const_defined?(:FORMAT) + FORMAT = '%Y-%m-%dT%H:%M:%S'.freeze + end + + def initialize(value, language: nil, datatype: nil, lexical: nil, validate: false, canonicalize: false, **options) + @object = value.freeze + @string = lexical if lexical + @string = value if !defined?(@string) && value.is_a?(String) + @string = @string.encode(Encoding::UTF_8).freeze if instance_variable_defined?(:@string) + @object = @string if instance_variable_defined?(:@string) && @object.is_a?(String) + @language = language.to_s.downcase.to_sym if language + @datatype = RDF::URI(datatype).freeze if datatype + @datatype ||= self.class.const_get(:DATATYPE) if self.class.const_defined?(:DATATYPE) + @datatype ||= instance_variable_defined?(:@language) && @language ? RDF.langString : RDF::URI("http://www.w3.org/2001/XMLSchema#string") + @original_datatype = datatype + end + + attr_reader :original_datatype end + end #end RDF diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb index 730f21de..9170d318 100644 --- a/lib/goo/search/search.rb +++ b/lib/goo/search/search.rb @@ -1,4 +1,5 @@ require 'rsolr' +require_relative 'solr/solr_connector' module Goo @@ -8,91 +9,189 @@ def self.included(base) base.extend(ClassMethods) end - def index(connection_name=:main) + def index(connection_name = nil, to_set = nil) raise ArgumentError, "ID must be set to be able to index" if @id.nil? - doc = indexable_object - Goo.search_connection(connection_name).add(doc) + document = indexable_object(to_set) + + return if document.blank? || document[:id].blank? + + connection_name ||= self.class.search_collection_name + unindex(connection_name) + self.class.search_client(connection_name)&.index_document(document) end - def index_update(to_set, connection_name=:main) + def index_update(attributes_to_update, connection_name = nil, to_set = nil) raise ArgumentError, "ID must be set to be able to index" if @id.nil? - raise ArgumentError, "Field names to be updated in index must be provided" if to_set.nil? + raise ArgumentError, "Field names to be updated in index must be provided" if attributes_to_update.blank? + + old_doc = self.class.search("id:\"#{index_id}\"").dig("response", "docs")&.first + + raise ArgumentError, "ID must be set to be able to index" if old_doc.blank? + doc = indexable_object(to_set) - doc.each { |key, val| - next if key === :id - doc[key] = {set: val} - } + doc.each do |key, val| + next unless attributes_to_update.any? { |attr| key.to_s.eql?(attr.to_s) || key.to_s.include?("#{attr}_") } + old_doc[key] = val + end + + connection_name ||= self.class.search_collection_name + unindex(connection_name) - Goo.search_connection(connection_name).update( - data: "[#{doc.to_json}]", - headers: { 'Content-Type' => 'application/json' } - ) + old_doc.reject! { |k, v| k.to_s.end_with?('_sort') || k.to_s.end_with?('_sorts') } + old_doc.delete("_version_") + self.class.search_client(connection_name).index_document(old_doc) end - def unindex(connection_name=:main) - id = index_id - Goo.search_connection(connection_name).delete_by_id(id) + def unindex(connection_name = nil) + connection_name ||= self.class.search_collection_name + self.class.search_client(connection_name)&.delete_by_id(index_id) end # default implementation, should be overridden by child class - def index_id() + def index_id raise ArgumentError, "ID must be set to be able to index" if @id.nil? @id.to_s end # default implementation, should be overridden by child class - def index_doc(to_set=nil) + def index_doc(to_set = nil) raise NoMethodError, "You must define method index_doc in your class for it to be indexable" end - def indexable_object(to_set=nil) - doc = index_doc(to_set) - # use resource_id for the actual term id because :id is a Solr reserved field - doc[:resource_id] = doc[:id].to_s - doc[:id] = index_id.to_s - doc + def embedded_doc + raise NoMethodError, "You must define method embedded_doc in your class for it to be indexable" end + def indexable_object(to_set = nil) + begin + document = index_doc(to_set) + rescue NoMethodError + document = self.to_hash.reject { |k, _| !self.class.indexable?(k) } + document.transform_values! do |v| + is_array = v.is_a?(Array) + v = Array(v).map do |x| + if x.is_a?(Goo::Base::Resource) + x.embedded_doc rescue x.id.to_s + else + if x.is_a?(RDF::URI) + x.to_s + else + x.respond_to?(:object) ? x.object : x + end + end + end + is_array ? v : v.first + end + end + + # Merge sub-objects into a single document. For example, OntologySubmission.ontology + # All Ontology object's properties are merged into a single document for OntologySubmission + # Solr datatypes are also added, i.e: + # {"name": "*_txt", "type": "text_general", stored: true, "multiValued": true} + # {"name": "*_text", "type": "text_general", stored: true, "multiValued": false }, + document = document.reduce({}) do |h, (k, v)| + if v.is_a?(Hash) + v.each { |k2, v2| h["#{k}_#{k2}".to_sym] = v2 } + else + h[k] = v + end + h + end + + model_name = self.class.model_name.to_s.downcase + document.delete(:id) + document.delete("id") + + document.transform_keys! do |k| + self.class.index_document_attr(k) + end + + document[:resource_id] = self.id.to_s + document[:resource_model] = model_name + document[:id] = index_id.to_s + document + end module ClassMethods - def search(q, params={}, connection_name=:main) - params["q"] = q - Goo.search_connection(connection_name).post('select', :data => params) + def index_enabled? + !@model_settings[:search_collection].nil? end - def indexBatch(collection, connection_name=:main) - docs = Array.new - collection.each do |c| - docs << c.indexable_object + def enable_indexing(collection_name, search_backend = :main, &block) + @model_settings[:search_collection] = collection_name + + if block_given? + # optional block to generate custom schema + Goo.add_search_connection(collection_name, search_backend, &block) + else + Goo.add_search_connection(collection_name, search_backend) end - Goo.search_connection(connection_name).add(docs) + + after_save :index + after_destroy :unindex end - def unindexBatch(collection, connection_name=:main) - docs = Array.new - collection.each do |c| - docs << c.index_id - end - Goo.search_connection(connection_name).delete_by_id(docs) + def search_collection_name + @model_settings[:search_collection] + end + + def search_client(connection_name = search_collection_name) + Goo.search_client(connection_name) + end + + def custom_schema?(connection_name = search_collection_name) + search_client(connection_name)&.custom_schema? + end + + def schema_generator + Goo.search_client(search_collection_name).schema_generator + end + + def index_document_attr(key) + return key.to_s if custom_schema? || self.attribute_settings(key).nil? + + type = self.datatype(key) + is_list = self.list?(key) + fuzzy = self.fuzzy_searchable?(key) + + SOLR::SolrConnector.index_document_attr(key, type, is_list, fuzzy) + end + + def search(q, params = {}, connection_name = search_collection_name) + search_client(connection_name).search(q, params) + end + + def submit_search_query(query, params = {}, connection_name = search_collection_name) + search_client(connection_name).submit_search_query(query, params) + end + + def indexBatch(collection, connection_name = search_collection_name) + docs = collection.map(&:indexable_object) + search_client(connection_name).index_document(docs) + end + + def unindexBatch(collection, connection_name = search_collection_name) + docs = collection.map(&:index_id) + search_client(connection_name).delete_by_id(docs) end - def unindexByQuery(query, connection_name=:main) - Goo.search_connection(connection_name).delete_by_query(query) + def unindexByQuery(query, connection_name = search_collection_name) + search_client(connection_name)&.delete_by_query(query) end - def indexCommit(attrs=nil, connection_name=:main) - Goo.search_connection(connection_name).commit(:commit_attributes => attrs || {}) + def indexCommit(attrs = nil, connection_name = search_collection_name) + search_client(connection_name)&.index_commit(attrs) end - def indexOptimize(attrs=nil, connection_name=:main) - Goo.search_connection(connection_name).optimize(:optimize_attributes => attrs || {}) + def indexOptimize(attrs = nil, connection_name = search_collection_name) + search_client(connection_name).index_optimize(attrs) end - def indexClear(connection_name=:main) - # WARNING: this deletes ALL data from the index - unindexByQuery("*:*", connection_name) + # WARNING: this deletes ALL data from the index + def indexClear(connection_name = search_collection_name) + search_client(connection_name).clear_all_data end end end diff --git a/lib/goo/search/solr/solr_admin.rb b/lib/goo/search/solr/solr_admin.rb new file mode 100644 index 00000000..4d20271b --- /dev/null +++ b/lib/goo/search/solr/solr_admin.rb @@ -0,0 +1,79 @@ +module SOLR + module Administration + + def admin_url + "#{@solr_url}/admin" + end + + def solr_alive? + collections_url = URI.parse("#{admin_url}/collections?action=CLUSTERSTATUS") + http = Net::HTTP.new(collections_url.host, collections_url.port) + request = Net::HTTP::Get.new(collections_url.request_uri) + + begin + response = http.request(request) + return response.code.eql?("200") && JSON.parse(response.body).dig("responseHeader", "status").eql?(0) + rescue StandardError => e + return false + end + end + + def fetch_all_collections + collections_url = URI.parse("#{admin_url}/collections?action=LIST") + + http = Net::HTTP.new(collections_url.host, collections_url.port) + request = Net::HTTP::Get.new(collections_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to fetch collections. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to fetch collections. #{e.message}" + end + + collections = [] + if response.is_a?(Net::HTTPSuccess) + collections = JSON.parse(response.body)['collections'] + end + + collections + end + + def create_collection(name = @collection_name, num_shards = 1, replication_factor = 1) + return if collection_exists?(name) + create_collection_url = URI.parse("#{admin_url}/collections?action=CREATE&name=#{name}&numShards=#{num_shards}&replicationFactor=#{replication_factor}") + + http = Net::HTTP.new(create_collection_url.host, create_collection_url.port) + request = Net::HTTP::Post.new(create_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to create collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to create collection. #{e.message}" + end + end + + def delete_collection(collection_name = @collection_name) + return unless collection_exists?(collection_name) + + delete_collection_url = URI.parse("#{admin_url}/collections?action=DELETE&name=#{collection_name}") + + http = Net::HTTP.new(delete_collection_url.host, delete_collection_url.port) + request = Net::HTTP::Post.new(delete_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to delete collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to delete collection. #{e.message}" + end + + end + + def collection_exists?(collection_name) + fetch_all_collections.include?(collection_name.to_s) + end + end +end + diff --git a/lib/goo/search/solr/solr_connector.rb b/lib/goo/search/solr/solr_connector.rb new file mode 100644 index 00000000..e367f5cd --- /dev/null +++ b/lib/goo/search/solr/solr_connector.rb @@ -0,0 +1,41 @@ +require 'rsolr' +require_relative 'solr_schema_generator' +require_relative 'solr_schema' +require_relative 'solr_admin' +require_relative 'solr_query' + +module SOLR + + class SolrConnector + include Schema, Administration, Query + attr_reader :solr + + def initialize(solr_url, collection_name) + @solr_url = solr_url + @collection_name = collection_name + @solr = RSolr.connect(url: collection_url) + + # Perform a status test and wait up to 30 seconds before raising an error + wait_time = 0 + max_wait_time = 30 + until solr_alive? || wait_time >= max_wait_time + sleep 1 + wait_time += 1 + end + raise "Solr instance not reachable within #{max_wait_time} seconds" unless solr_alive? + + + @custom_schema = false + end + + def init(force = false) + return if collection_exists?(@collection_name) && !force + + create_collection + + init_schema + end + + end +end + diff --git a/lib/goo/search/solr/solr_query.rb b/lib/goo/search/solr/solr_query.rb new file mode 100644 index 00000000..b11dac3e --- /dev/null +++ b/lib/goo/search/solr/solr_query.rb @@ -0,0 +1,110 @@ +module SOLR + module Query + + def self.included(base) + base.extend(ClassMethods) + end + + module ClassMethods + def index_document_attr(key, type, is_list, fuzzy_search) + dynamic_field(type: type, is_list: is_list, is_fuzzy_search: fuzzy_search).gsub('*', key.to_s) + end + + private + + def dynamic_field(type:, is_list:, is_fuzzy_search: false) + return is_list ? '*_texts' : '*_text' if is_fuzzy_search + + dynamic_type = case type + when :uri, :url, :string, nil + '*_t' + when :integer + '*_i' + when :boolean + '*_b' + when :date_time + '*_dt' + when :float + '*_f' + else + # Handle unknown data types or raise an error based on your specific requirements + raise ArgumentError, "Unsupported ORM data type: #{type}" + end + + if is_list + dynamic_type = dynamic_type.eql?('*_t') ? "*_txt" : "#{dynamic_type}s" + end + + dynamic_type + end + end + + def clear_all_data + delete_by_query('*:*') + end + + def collection_url + "#{@solr_url}/#{@collection_name}" + end + + def index_commit(attrs = nil) + @solr.commit(:commit_attributes => attrs || {}) + end + + def index_optimize(attrs = nil) + @solr.optimize(:optimize_attributes => attrs || {}) + end + + def index_document(document, commit: true) + @solr.add(document) + @solr.commit if commit + end + + def index_document_attr(key, type, is_list, fuzzy_search) + self.class.index_document_attr(key, type, is_list, fuzzy_search) + end + + + + def delete_by_id(document_id, commit: true) + return if document_id.nil? + + @solr.delete_by_id(document_id) + @solr.commit if commit + end + + def delete_by_query(query) + @solr.delete_by_query(query) + @solr.commit + end + + def search(query, params = {}) + body = params.dup + body[:q] = query + @solr.post('select', data: body) + end + + def submit_search_query(query, params = {}) + uri = ::URI.parse("#{collection_url}/select") + http = Net::HTTP.new(uri.host, uri.port) + request = Net::HTTP::Post.new(uri.request_uri) + + params = params.dup + params[:q] = query + params[:wt] ||= "json" + + request["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8" + request["X-Requested-With"] = "XMLHttpRequest" # helps when Solr CSRF filter is enabled + request.set_form_data(params) + + response = http.request(request) + + unless response.is_a?(Net::HTTPSuccess) + raise "Solr query failed (HTTP #{response.code} #{response.message}): #{response.body}" + end + + JSON.parse(response.body) + end + + end +end diff --git a/lib/goo/search/solr/solr_schema.rb b/lib/goo/search/solr/solr_schema.rb new file mode 100644 index 00000000..84471165 --- /dev/null +++ b/lib/goo/search/solr/solr_schema.rb @@ -0,0 +1,184 @@ +module SOLR + module Schema + + def fetch_schema + uri = URI.parse("#{@solr_url}/#{@collection_name}/schema") + http = Net::HTTP.new(uri.host, uri.port) + + request = Net::HTTP::Get.new(uri.path, 'Content-Type' => 'application/json') + response = http.request(request) + + if response.code.to_i == 200 + @schema = JSON.parse(response.body)["schema"] + else + raise StandardError, "Failed to upload schema. HTTP #{response.code}: #{response.body}" + end + end + + def schema + @schema ||= fetch_schema + end + + def all_fields + schema["fields"] + end + + def all_copy_fields + schema["copyFields"] + end + + def all_dynamic_fields + schema["dynamicFields"] + end + + def all_fields_types + schema["fieldTypes"] + end + + def fetch_all_fields + fetch_schema["fields"] + end + + def fetch_all_copy_fields + fetch_schema["copyFields"] + end + + def fetch_all_dynamic_fields + fetch_schema["dynamicFields"] + end + + def fetch_all_fields_types + fetch_schema["fieldTypes"] + end + + def schema_generator + @schema_generator ||= SolrSchemaGenerator.new + end + + def init_collection(num_shards = 1, replication_factor = 1) + create_collection_url = URI.parse("#{@solr_url}/admin/collections?action=CREATE&name=#{@collection_name}&numShards=#{num_shards}&replicationFactor=#{replication_factor}") + + http = Net::HTTP.new(create_collection_url.host, create_collection_url.port) + request = Net::HTTP::Post.new(create_collection_url.request_uri) + + begin + response = http.request(request) + raise StandardError, "Failed to create collection. HTTP #{response.code}: #{response.message}" unless response.code.to_i == 200 + rescue StandardError => e + raise StandardError, "Failed to create collection. #{e.message}" + end + end + + def init_schema(generator = schema_generator) + clear_all_schema(generator) + fetch_schema + default_fields = all_fields.map { |f| f['name'] } + + solr_schema = { + "add-field-type": generator.field_types_to_add, + 'add-field' => generator.fields_to_add.reject { |f| default_fields.include?(f[:name]) }, + 'add-dynamic-field' => generator.dynamic_fields_to_add, + 'add-copy-field' => generator.copy_fields_to_add + } + + update_schema(solr_schema) + end + + def custom_schema? + @custom_schema + end + + def enable_custom_schema + @custom_schema = true + end + + def clear_all_schema(generator = schema_generator) + init_ft = generator.field_types_to_add.map { |f| f[:name] } + dynamic_fields = all_dynamic_fields.map { |f| { name: f['name'] } } + copy_fields = all_copy_fields.map { |f| { source: f['source'], dest: f['dest'] } } + fields_types = all_fields_types.select { |f| init_ft.include?(f['name']) }.map { |f| { name: f['name']} } + fields = all_fields.reject { |f| %w[id _version_ ].include?(f['name']) }.map { |f| { name: f['name'] } } + + upload_schema('delete-copy-field' => copy_fields) unless copy_fields.empty? + upload_schema('delete-dynamic-field' => dynamic_fields) unless dynamic_fields.empty? + upload_schema('delete-field' => fields) unless copy_fields.empty? + upload_schema('delete-field-type' => fields_types) unless fields_types.empty? + end + + def map_to_indexer_type(orm_data_type) + case orm_data_type + when :uri, :url + 'string' # Assuming a string field for URIs + when :string, nil # Default to 'string' if no type is given + 'text_general' # Assuming a generic text field for strings + when :integer + 'pint' + when :boolean + 'boolean' + when :date_time + 'pdate' + when :float + 'pfloat' + else + # Handle unknown data types or raise an error based on your specific requirements + raise ArgumentError, "Unsupported ORM data type: #{orm_data_type}" + end + end + + def delete_field(name) + update_schema('delete-field' => [ + { name: name } + ]) + end + + def add_field(name, type, indexed: true, stored: true, multi_valued: false) + update_schema('add-field' => [ + { name: name, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + ]) + end + + def add_dynamic_field(name, type, indexed: true, stored: true, multi_valued: false) + update_schema('add-dynamic-field' => [ + { name: name, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + ]) + end + + def add_copy_field(source, dest) + update_schema('add-copy-field' => [ + { source: source, dest: dest } + ]) + end + + def fetch_field(name) + fetch_all_fields.select { |f| f['name'] == name }.first + end + + def update_schema(schema_json) + permitted_actions = %w[add-field add-copy-field add-dynamic-field add-field-type delete-copy-field delete-dynamic-field delete-field delete-field-type] + + unless permitted_actions.any? { |action| schema_json.key?(action) } + raise StandardError, "The schema need to implement at least one of this actions: #{permitted_actions.join(', ')}" + end + upload_schema(schema_json) + fetch_schema + end + + private + + def upload_schema(schema_json) + uri = URI.parse("#{@solr_url}/#{@collection_name}/schema") + http = Net::HTTP.new(uri.host, uri.port) + + request = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json') + request.body = schema_json.to_json + response = http.request(request) + if response.code.to_i == 200 + response + else + raise StandardError, "Failed to upload schema. HTTP #{response.code}: #{response.body}" + end + end + + end +end + diff --git a/lib/goo/search/solr/solr_schema_generator.rb b/lib/goo/search/solr/solr_schema_generator.rb new file mode 100644 index 00000000..c36e4f50 --- /dev/null +++ b/lib/goo/search/solr/solr_schema_generator.rb @@ -0,0 +1,327 @@ +module SOLR + + class SolrSchemaGenerator + + attr_reader :schema + + def initialize + @schema = {} + end + + def add_field(name, type, indexed: true, stored: true, multi_valued: false, omit_norms: nil, default: nil) + @schema['add-field'] ||= [] + af = { name: name.to_s, type: type, indexed: indexed, stored: stored, multiValued: multi_valued} + af[:omitNorms] = omit_norms unless omit_norms.nil? + # Solr Schema API expects `default` as a STRING value; sending JSON booleans/numbers + # can trigger ClassCastException on Solr side. + af[:default] = default.is_a?(String) ? default : default.to_s unless default.nil? + @schema['add-field'] << af + end + + def add_dynamic_field(name, type, indexed: true, stored: true, multi_valued: false, omit_norms: nil) + @schema['add-dynamic-field'] ||= [] + df = { name: name.to_s, type: type, indexed: indexed, stored: stored, multiValued: multi_valued } + df[:omitNorms] = omit_norms unless omit_norms.nil? + @schema['add-dynamic-field'] << df + end + + def add_copy_field(source, dest) + @schema['add-copy-field'] ||= [] + @schema['add-copy-field'] << { source: source, dest: dest } + end + + def add_field_type(type_definition) + @schema['add-field-type'] ||= [] + @schema['add-field-type'] << type_definition + end + + def fields_to_add + custom_fields = @schema['add-field'] || [] + custom_fields + init_fields + end + + def dynamic_fields_to_add + custom_fields = @schema['add-dynamic-field'] || [] + custom_fields + init_dynamic_fields + end + + def copy_fields_to_add + custom_fields = @schema['add-copy-field'] || [] + custom_fields + init_copy_fields + end + + def field_types_to_add + custom_fields = @schema['add-field-type'] || [] + custom_fields + init_fields_types + end + + def init_fields_types + [ + { + "name": "string_ci", + "class": "solr.TextField", + "sortMissingLast": true, + "omitNorms": true, + "queryAnalyzer": + { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + } + ] + } + }, + { + "name": "text_suggest_ngram", + "class": "solr.TextField", + "positionIncrementGap": "100", + "indexAnalyzer": { + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "1", + "generateNumberParts": "1", + "catenateWords": "0", + "catenateNumbers": "0", + "catenateAll": "0", + "splitOnCaseChange": "1" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.ASCIIFoldingFilterFactory" + }, + { + "class": "solr.EdgeNGramFilterFactory", + "minGramSize": 1, + "maxGramSize": 20 + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": "", + "replace": "all" + } + ] + }, + "queryAnalyzer": { + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "0", + "generateNumberParts": "0", + "catenateWords": "0", + "catenateNumbers": "0", + "catenateAll": "0", + "splitOnCaseChange": "0" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.ASCIIFoldingFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": "", + "replace": "all" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "^(.{20})(.*)?", + "replacement": "$1", + "replace": "all" + } + ] + } + }, + { + "name": "text_suggest_edge", + "class": "solr.TextField", + "positionIncrementGap": "100", + "indexAnalyzer": { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([\\.,;:-_])", + "replacement": " ", + "replace": "all" + }, + { + "class": "solr.EdgeNGramFilterFactory", + "minGramSize": 1, + "maxGramSize": 30, + "preserveOriginal": true + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": "", + "replace": "all" + } + ] + }, + "queryAnalyzer": { + "tokenizer": { + "class": "solr.KeywordTokenizerFactory" + }, + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "filters": [ + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([\\.,;:-_])", + "replacement": " ", + "replace": "all" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": "", + "replace": "all" + } + ] + } + }, + { + "name": "text_suggest", + "class": "solr.TextField", + "positionIncrementGap": 100, + indexAnalyzer: { + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "1", + "generateNumberParts": "1", + "catenateWords": "1", + "catenateNumbers": "1", + "catenateAll": "1", + "splitOnCaseChange": "1", + "splitOnNumerics": "1", + "preserveOriginal": "1" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": " ", + "replace": "all" + } + ] + }, + queryAnalyzer: { + "char_filters": [ + { + "class": "solr.MappingCharFilterFactory", + "mapping": "solr/resources/org/apache/lucene/analysis/miscellaneous/MappingCharFilter.greekaccent" + } + ], + "tokenizer": { + "class": "solr.StandardTokenizerFactory" + }, + "filters": [ + { + "class": "solr.WordDelimiterGraphFilterFactory", + "generateWordParts": "0", + "generateNumberParts": "0", + "catenateWords": "0", + "catenateNumbers": "0", + "catenateAll": "0", + "splitOnCaseChange": "0" + }, + { + "class": "solr.LowerCaseFilterFactory" + }, + { + "class": "solr.PatternReplaceFilterFactory", + "pattern": "([^\\p{L}\\p{N}\\* ])", + "replacement": " ", + "replace": "all" + } + ] + } + } + ] + end + + def init_fields + [ + #{ name: "_version_", type: "plong", indexed: true, stored: true, multiValued: false }, + { name: "resource_id", type: "string", indexed: true, multiValued: false, required: true, stored: true }, + { name: "resource_model", type: "string", indexed: true, multiValued: false, required: true, stored: false }, + { name: "_text_", type: "text_general", indexed: true, multiValued: true, stored: false }, + ] + end + + def init_dynamic_fields + [ + {"name": "*_t", "type": "text_general", stored: true, "multiValued": false }, + {"name": "*_txt", "type": "text_general", stored: true, "multiValued": true}, + {"name": "*_i", "type": "pint", stored: true }, + {"name": "*_is", "type": "pints", stored: true }, + {"name": "*_f", "type": "pfloat", stored: true }, + {"name": "*_fs", "type": "pfloats", stored: true }, + {"name": "*_b", "type": "boolean", stored: true }, + {"name": "*_bs", "type": "booleans", stored: true }, + {"name": "*_dt", "type": "pdate", stored: true }, + {"name": "*_dts", "type": "pdate", stored: true , multiValued: true}, + { "name": "*Exact", "type": "string_ci", "multiValued": true, stored: false }, + { "name": "*Suggest", "type": "text_suggest", "omitNorms": true, stored: false, "multiValued": true }, + { "name": "*SuggestEdge", "type": "text_suggest_edge", stored: false, "multiValued": true }, + { "name": "*SuggestNgram", "type": "text_suggest_ngram", stored: false, "omitNorms": true, "multiValued": true }, + { "name": "*_text", "type": "text_general", stored: true, "multiValued": false }, + { "name": "*_texts", "type": "text_general", stored: true, "multiValued": true }, + {"name": "*_sort", "type": "string", stored: false }, + {"name": "*_sorts", "type": "strings", stored: false , "multiValued": true}, + ] + end + + def init_copy_fields + [ + { source: "*_text", dest: %w[_text_ *Exact *Suggest *SuggestEdge *SuggestNgram *_sort] }, + { source: "*_texts", dest: %w[_text_ *Exact *Suggest *SuggestEdge *SuggestNgram *_sorts] }, + ] + end + end +end diff --git a/lib/goo/sparql/client.rb b/lib/goo/sparql/client.rb index 35b60b5f..5ac67ed4 100644 --- a/lib/goo/sparql/client.rb +++ b/lib/goo/sparql/client.rb @@ -14,8 +14,6 @@ class Client < RSPARQL::Client "text/x-nquads" => "nquads" } - BACKEND_4STORE = "4store" - def status_based_sleep_time(operation) sleep(0.5) st = self.status @@ -39,85 +37,105 @@ def status_based_sleep_time(operation) end class DropGraph - def initialize(g) + def initialize(g, silent: false) @graph = g @caching_options = { :graph => @graph.to_s } + @silent = silent end + def to_s - return "DROP GRAPH <#{@graph.to_s}>" + "DROP #{@silent ? 'SILENT' : ''} GRAPH <#{@graph.to_s}>" end + def options - #Returns the caching option - return @caching_options + # Returns the caching option + @caching_options end end - def bnodes_filter_file(file_path,mime_type) + def bnodes_filter_file(file_path, mime_type) mime_type = "application/rdf+xml" if mime_type.nil? format = MIMETYPE_RAPPER_MAP[mime_type] if format.nil? raise Exception, "mime_type #{mime_type} not supported in slicing" end dir = Dir.mktmpdir("file_nobnodes") - dst_path = File.join(dir,"data.nt") - dst_path_bnodes_out = File.join(dir,"data_no_bnodes.nt") + dst_path = File.join(dir, "data.nt") + dst_path_bnodes_out = File.join(dir, "data_no_bnodes.nt") out_format = format == "nquads" ? "nquads" : "ntriples" rapper_command_call = "rapper -i #{format} -o #{out_format} #{file_path} > #{dst_path}" - stdout,stderr,status = Open3.capture3(rapper_command_call) + stdout, stderr, status = Open3.capture3(rapper_command_call) if not status.success? raise Exception, "Rapper cannot parse #{format} file at #{file_path}: #{stderr}" end filter_command = "LANG=C grep -v '_:genid' #{dst_path} > #{dst_path_bnodes_out}" - stdout,stderr,status = Open3.capture3(filter_command) + stdout, stderr, status = Open3.capture3(filter_command) if not status.success? raise Exception, "could not `#{filter_command}`: #{stderr}" end - return dst_path_bnodes_out,dir + return dst_path_bnodes_out, dir end def delete_data_graph(graph) - Goo.sparql_update_client.update(DropGraph.new(graph)) + Goo.sparql_update_client.update(DropGraph.new(graph, silent: Goo.backend_vo?)) end - def append_triples_no_bnodes(graph,file_path,mime_type_in) - bnodes_filter = nil - dir = nil + def append_triples_batch(graph, triples, mime_type_in, current_line = 0) + begin + puts "Appending triples in batch of #{triples.size} triples from line #{current_line}" + execute_append_request graph, triples.join, mime_type_in + rescue RestClient::Exception => e + puts "Error in appending triples request: #{e.response}" + if triples.size < 100 + triples.each_with_index do |line, i| + begin + execute_append_request graph, line, mime_type_in + rescue RestClient::Exception => e + puts "Error in append request: #{e.response} line #{i + current_line}: #{line}" + end + end + else + half = triples.size / 2 + append_triples_batch(graph, triples[0..half], mime_type_in, current_line) + append_triples_batch(graph, triples[half..-1], mime_type_in, current_line + half) + end - if file_path.end_with?("ttl") + end + end + + def append_triples_no_bnodes(graph, file_path, mime_type_in) + dir = nil + response = nil + if file_path.end_with?('ttl') || file_path.end_with?('nt') || file_path.end_with?('n3') bnodes_filter = file_path else - bnodes_filter,dir = bnodes_filter_file(file_path,mime_type_in) + bnodes_filter, dir = bnodes_filter_file(file_path, mime_type_in) end - mime_type = "text/turtle" - if mime_type_in == "text/x-nquads" - mime_type = "text/x-nquads" - graph = "http://data.bogus.graph/uri" + if Goo.backend_vo? || Goo.backend_ag? + chunk_lines = 50_000 # number of line + else + chunk_lines = 500_000 # number of line end - data_file = File.read(bnodes_filter) - params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil} - backend_name = Goo.sparql_backend_name + file = File.foreach(bnodes_filter) + lines = [] + line_count = 0 + file.each_entry do |line| + lines << line - if backend_name == BACKEND_4STORE - params[:payload] = { - graph: graph.to_s, - data: data_file, - "mime-type" => mime_type - } - #for some reason \\\\ breaks parsing - params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n") - else - params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}" - params[:payload] = data_file + if lines.size == chunk_lines + response = append_triples_batch(graph, lines, mime_type_in, line_count) + line_count += lines.size + lines.clear + end end - response = RestClient::Request.execute(params) + response = append_triples_batch(graph, lines, mime_type_in, line_count) unless lines.empty? unless dir.nil? File.delete(bnodes_filter) - begin FileUtils.rm_rf(dir) rescue => e @@ -128,43 +146,43 @@ def append_triples_no_bnodes(graph,file_path,mime_type_in) response end - def append_data_triples(graph,data,mime_type) + def append_data_triples(graph, data, mime_type) f = Tempfile.open('data_triple_store') f.write(data) f.close() - res = append_triples_no_bnodes(graph,f.path,mime_type) + res = append_triples_no_bnodes(graph, f.path, mime_type) return res end - def put_triples(graph,file_path,mime_type=nil) + def put_triples(graph, file_path, mime_type = nil) delete_graph(graph) - result = append_triples_no_bnodes(graph,file_path,mime_type) - Goo.sparql_query_client.cache_invalidate_graph(graph) + result = append_triples_no_bnodes(graph, file_path, mime_type) + Goo.sparql_query_client.cache.invalidate(graph) result end - def append_triples(graph,data,mime_type=nil) - result = append_data_triples(graph,data,mime_type) - Goo.sparql_query_client.cache_invalidate_graph(graph) + def append_triples(graph, data, mime_type = nil) + result = append_data_triples(graph, data, mime_type) + Goo.sparql_query_client.cache.invalidate(graph) result end - def append_triples_from_file(graph,file_path,mime_type=nil) + def append_triples_from_file(graph, file_path, mime_type = nil) if mime_type == "text/nquads" && !graph.instance_of?(Array) raise Exception, "Nquads need a list of graphs, #{graph} provided" end - result = append_triples_no_bnodes(graph,file_path,mime_type) - Goo.sparql_query_client.cache_invalidate_graph(graph) + result = append_triples_no_bnodes(graph, file_path, mime_type) + Goo.sparql_query_client.cache.invalidate(graph) result end def delete_graph(graph) result = delete_data_graph(graph) - Goo.sparql_query_client.cache_invalidate_graph(graph) + Goo.sparql_query_client.cache.invalidate(graph) return result end - def extract_number_from(i,text) + def extract_number_from(i, text) res = [] while (text[i] != '<') res << text[i] @@ -180,7 +198,7 @@ def status resp_text = nil begin - resp_text = Net::HTTP.get(URI(status_url)) + resp_text = Net::HTTP.get(URI(status_url)) rescue StandardError => e resp[:exception] = "Error connecting to triple store: #{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}" return resp @@ -196,6 +214,38 @@ def status resp[:outstanding] = outstanding resp end + + def params_for_backend(graph, data_file, mime_type_in, method = :post) + mime_type = "text/turtle" + + if mime_type_in == "text/x-nquads" + mime_type = "text/x-nquads" + graph = "http://data.bogus.graph/uri" + end + + params = { method: method, url: "#{url.to_s}", headers: { "content-type" => mime_type, "mime-type" => mime_type }, timeout: nil } + + if Goo.backend_4s? + params[:payload] = { + graph: graph.to_s, + data: data_file, + 'mime-type' => mime_type + } + # for some reason \\\\ breaks parsing + params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\", "") }.join("\n") + elsif Goo.backend_vo? + params[:url] = "#{url.parent}/sparql-graph-crud?graph=#{CGI.escape(graph.to_s)}" + params[:payload] = data_file + else + params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}" + params[:payload] = data_file + end + params + end + + def execute_append_request(graph, data_file, mime_type_in) + RestClient::Request.execute(params_for_backend(graph, data_file, mime_type_in)) + end end end end diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb index 821aba26..cc101855 100644 --- a/lib/goo/sparql/loader.rb +++ b/lib/goo/sparql/loader.rb @@ -1,3 +1,4 @@ +require 'request_store' module Goo module SPARQL module Loader @@ -6,8 +7,10 @@ class << self def model_load(*options) options = options.last + set_request_lang(options) if options[:models] && options[:models].is_a?(Array) && \ (options[:models].length > Goo.slice_loading_size) + options = options.dup models = options[:models] include_options = options[:include] @@ -33,78 +36,56 @@ def model_load(*options) ## def model_load_sliced(*options) options = options.last - ids = options[:ids] klass = options[:klass] incl = options[:include] models = options[:models] - aggregate = options[:aggregate] - read_only = options[:read_only] collection = options[:collection] - count = options[:count] - include_pagination = options[:include_pagination] - equivalent_predicates = options[:equivalent_predicates] - predicates = options[:predicates] - - embed_struct, klass_struct = get_structures(aggregate, count, incl, include_pagination, klass, read_only) - raise_resource_must_persistent_error(models) if models + embed_struct, klass_struct = get_structures(options[:aggregate], options[:count] , incl, options[:include_pagination], klass, options[:read_only]) + raise_not_persistent_error(models) if models graphs = get_graphs(collection, klass) - ids, models_by_id = get_models_by_id_hash(ids, klass, klass_struct, models) + models_by_id = get_models_by_id_hash( options[:ids], klass, klass_struct, models) - query_options = {} #TODO: breaks the reasoner patterns = [[:id, RDF.type, klass.uri_type(collection)]] incl_embed = nil - unmapped = nil bnode_extraction = nil properties_to_include = [] variables = [:id] - if incl - if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode) + if incl && !incl.empty? + if incl.first.is_a?(Hash) && incl.first.include?(:bnode) #limitation only one level BNODE bnode_extraction, patterns, variables = get_bnode_extraction(collection, incl, klass, patterns) else variables = %i[id attributeProperty attributeObject] if incl.first == :unmapped - unmapped = true - properties_to_include = predicate_map(predicates) + properties_to_include = predicate_map(options[:predicates]) else - #make it deterministic - incl_embed = get_embed_includes(incl) - graphs, properties_to_include, query_options = get_includes(collection, graphs, incl, - klass, query_options) + graphs, properties_to_include, incl_embed = get_includes(collection, graphs, incl, klass) end end end - expand_equivalent_predicates(properties_to_include, equivalent_predicates) - query_builder = Goo::SPARQL::QueryBuilder.new options - select, aggregate_projections = query_builder.build_select_query(ids, variables, graphs, - patterns, query_options, - properties_to_include) + options[:properties_to_include] = properties_to_include + + + select, aggregate_projections = Goo::SPARQL::QueryBuilder.new(options) + .build_query(models_by_id.keys, variables, graphs, patterns) solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction, embed_struct, incl_embed, klass_struct, models_by_id, - properties_to_include, unmapped, - variables, ids, options + variables, options solution_mapper.map_each_solutions(select) end private - def expand_equivalent_predicates(properties_to_include, eq_p) - - return unless eq_p && !eq_p.empty? - - properties_to_include&.each do |property_attr, property| - property_uri = property[:uri] - property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s) - end - + def set_request_lang(options) + options[:requested_lang] = RequestStore.store[:requested_lang] end def predicate_map(predicates) @@ -114,10 +95,11 @@ def predicate_map(predicates) predicates_map = {} uniq_p.each do |p| i = 0 - key = ('var_' + p.last_part + i.to_s).to_sym + last_part = p.to_s.include?("#") ? p.to_s.split('#').last : p.to_s.split('/').last + key = ('var_' + last_part + i.to_s).to_sym while predicates_map.include?(key) i += 1 - key = ('var_' + p.last_part + i.to_s).to_sym + key = ('var_' + last_part + i.to_s).to_sym break if i > 10 end predicates_map[key] = { uri: p, is_inverse: false } @@ -126,19 +108,19 @@ def predicate_map(predicates) predicates_map end - def get_includes(collection, graphs, incl, klass, query_options) + def get_includes(collection, graphs, incl, klass) + incl_embed ,incl = get_embed_includes(incl) incl = incl.to_a incl.delete_if { |a| !a.instance_of?(Symbol) } properties_to_include = {} incl.each do |attr| graph, pattern = query_pattern(klass, attr, collection: collection) - add_rules(attr, klass, query_options) if klass.attributes(:all).include?(attr) properties_to_include[attr] = { uri: pattern[1], is_inverse: klass.inverse?(attr) } # [property_attr, property_uri , inverse: true] end graphs << graph if graph && (!klass.collection_opts || klass.inverse?(attr)) end - [graphs, properties_to_include,query_options] + [graphs, properties_to_include, incl_embed] end def get_bnode_extraction(collection, incl, klass, patterns) @@ -175,7 +157,7 @@ def get_models_by_id_hash(ids, klass, klass_struct, models) #a where without models end - return ids, models_by_id + models_by_id end def get_graphs(collection, klass) @@ -228,7 +210,7 @@ def get_structures(aggregate, count, incl, include_pagination, klass, read_only) [embed_struct, klass_struct] end - def raise_resource_must_persistent_error(models) + def raise_not_persistent_error(models) models.each do |m| if (not m.nil?) && !m.respond_to?(:klass) #read only raise ArgumentError, @@ -246,7 +228,7 @@ def get_embed_includes(incl) #variables.concat(embed_variables) incl.concat(embed_variables) end - incl_embed + [incl_embed, incl] end end diff --git a/lib/goo/sparql/mixins/query_pattern.rb b/lib/goo/sparql/mixins/query_pattern.rb index cc370795..9ee0df7d 100644 --- a/lib/goo/sparql/mixins/query_pattern.rb +++ b/lib/goo/sparql/mixins/query_pattern.rb @@ -3,9 +3,6 @@ module SPARQL module QueryPatterns - def add_rules(attr,klass,query_options) - (query_options[:rules] ||= []) << :SUBC if klass.transitive?(attr) - end def query_pattern(klass,attr,**opts) value = opts[:value] || nil diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb new file mode 100644 index 00000000..cf97fd07 --- /dev/null +++ b/lib/goo/sparql/mixins/solution_lang_filter.rb @@ -0,0 +1,186 @@ +module Goo + module SPARQL + module Solution + class LanguageFilter + + attr_reader :requested_lang, :unmapped, :objects_by_lang + + def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: []) + @list_attributes = list_attributes + @objects_by_lang = {} + @unmapped = unmapped + @requested_lang = get_language(requested_lang) + end + + def fill_models_with_all_languages(models_by_id) + objects_by_lang.each do |id, predicates| + + model = models_by_id[id] + predicates.each do |predicate, values| + + if values.values.any? { |v| v.any? { |x| literal?(x) && x.plain?} } + + pull_stored_values(model, values, predicate, @unmapped) + end + end + end + end + + + def set_model_value(model, predicate, values, value) + set_value(model, predicate, value) do + model.send("#{predicate}=", values, on_load: true) + end + end + + def set_unmapped_value(model, predicate, value) + set_value(model, predicate, value) do + return add_unmapped_to_model(model, predicate, value) + end + end + + def models_unmapped_to_array(m) + if show_all_languages? + model_group_by_lang(m) + else + m.unmmaped_to_array + end + end + + private + + + def set_value(model, predicate, value, &block) + language = object_language(value) + + if requested_lang.eql?(:ALL) || !literal?(value) || (language_match?(language) && can_add_new_value?(model,predicate, language)) + block.call + end + + if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array) + language = "@none" if no_lang?(language) + store_objects_by_lang(model.id, predicate, value, language) + end + end + + + def can_add_new_value?(model, predicate, new_language) + old_val = model.send(predicate) rescue nil + list_attributes?(predicate) || old_val.blank? || !no_lang?(new_language) + end + + def no_lang?(language) + language.nil? || language.eql?(:no_lang) + end + + def model_group_by_lang(model) + unmapped = model.unmapped + cpy = {} + + unmapped.each do |attr, v| + cpy[attr] = group_by_lang(v) + end + + model.unmapped = cpy + end + + def group_by_lang(values) + + return values.to_a if values.all?{|x| x.is_a?(RDF::URI) || !x.respond_to?(:language) } + + values = values.group_by { |x| x.respond_to?(:language) && x.language ? x.language.to_s.downcase.to_sym : "@none" } + + no_lang = values["@none"] || [] + return no_lang if !no_lang.empty? && no_lang.all? { |x| x.respond_to?(:plain?) && !x.plain? } + + values + end + + + def object_language(new_value) + new_value.language || :no_lang if new_value.is_a?(RDF::Literal) + end + + def language_match?(language) + # no_lang means that the object is not a literal + return true if language.eql?(:no_lang) + + return requested_lang.include?(language.upcase) if requested_lang.is_a?(Array) + + language.upcase.eql?(requested_lang) + end + + def literal?(object) + !object_language(object).nil? + end + + def store_objects_by_lang(id, predicate, object, language) + # store objects in this format: [id][predicate][language] = [objects] + return if requested_lang.is_a?(Array) && !requested_lang.include?(language.upcase) && !language.eql?('@none') + + language_key = language.downcase + + objects_by_lang[id] ||= {} + objects_by_lang[id][predicate] ||= {} + objects_by_lang[id][predicate][language_key] ||= [] + + objects_by_lang[id][predicate][language_key] << object + end + + + def add_unmapped_to_model(model, predicate, value) + + if model.respond_to? :klass # struct + model[:unmapped] ||= {} + model[:unmapped][predicate] ||= [] + model[:unmapped][predicate] << value unless value.nil? + else + model.unmapped_set(predicate, value) + end + end + + def pull_stored_values(model, values, predicate, unmapped) + if unmapped + add_unmapped_to_model(model, predicate, values) + else + values = values.map do |language, values_literals| + values_string = values_literals.select{|x| literal?(x) && x.plain?}.map{|x| x.object} + values_string = values_string.first unless list_attributes?(predicate) + [language, values_string] + end.to_h.reject { |_key, value| value.empty? } + + model.send("#{predicate}=", values, on_load: true) + end + + end + + def unmapped_get(model, predicate) + if model && model.respond_to?(:klass) # struct + model[:unmapped]&.dig(predicate) + else + model.unmapped_get(predicate) + end + + end + + def list_attributes?(predicate) + @list_attributes.include?(predicate) + end + + + def show_all_languages? + @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL) + end + + def get_language(languages) + languages = Goo.portal_language if languages.nil? || languages.empty? + lang = languages + lang = languages.to_s.split(',') unless lang.is_a?(Array) + lang = lang.map { |l| l.upcase.to_sym } + lang.length == 1 ? lang.first : lang + end + + end + end + end +end diff --git a/lib/goo/sparql/processor.rb b/lib/goo/sparql/processor.rb new file mode 100644 index 00000000..c89778ed --- /dev/null +++ b/lib/goo/sparql/processor.rb @@ -0,0 +1,137 @@ +module Goo + module SPARQL + module Processor + def process_query_call(count=false) + if Goo.queries_debug? && Thread.current[:ncbo_debug] + start = Time.now + query_resp = process_query_intl(count=count) + (Thread.current[:ncbo_debug][:goo_process_query] ||= []) << (Time.now - start) + return query_resp + end + process_query_init(count=count) + end + + private + def process_query_init(count=false) + if @models == [] + @result = [] + return @result + end + + @include << @include_embed if @include_embed.length > 0 + + @predicates = unmmaped_predicates() + @equivalent_predicates = retrieve_equivalent_predicates() + + options_load = { models: @models, include: @include, ids: @ids, + graph_match: @pattern, klass: @klass, + filters: @filters, order_by: @order_by , + read_only: @read_only, rules: @rules, + predicates: @predicates, + no_graphs: @no_graphs, + equivalent_predicates: @equivalent_predicates } + + options_load.merge!(@where_options_load) if @where_options_load + + if !@klass.collection_opts.nil? and !options_load.include?(:collection) + raise ArgumentError, "Collection needed call `#{@klass.name}`" + end + + ids = nil + + + ids = redis_indexed_ids if use_redis_index? + + if @page_i && !use_redis_index? + page_options = options_load.dup + page_options.delete(:include) + page_options[:include_pagination] = @include + page_options[:query_options] = @query_options + + @count = run_count_query(page_options) + page_options[:page] = { page_i: @page_i, page_size: @page_size } + + models_by_id = Goo::SPARQL::Queries.model_load(page_options) + options_load[:models] = models_by_id.values + #models give the constraint + options_load.delete :graph_match + elsif count + count_options = options_load.dup + count_options.delete(:include) + return run_count_query(count_options) + end + + if @indexing + #do not care about include values + @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values) + return @result + end + + options_load[:ids] = ids if ids + models_by_id = {} + + if (@page_i && options_load[:models].nil?) || + (@page_i && options_load[:models].length > 0) || + (!@page_i && (@count.nil? || @count > 0)) + + models_by_id = Goo::SPARQL::Queries.model_load(options_load) + run_aggregate_query(models_by_id) if @aggregate && models_by_id.length > 0 + end + + if @page_i + @result = Goo::Base::Page.new(@page_i, @page_size, @count, models_by_id.values) + else + @result = @models ? @models : models_by_id.values + end + @result + end + + + def use_redis_index? + @index_key + end + + def run_aggregate_query(models_by_id) + options_load_agg = { models: models_by_id.values, klass: @klass, + filters: @filters, read_only: @read_only, + aggregate: @aggregate, rules: @rules } + options_load_agg.merge!(@where_options_load) if @where_options_load + Goo::SPARQL::Queries.model_load(options_load_agg) + end + def run_count_query(page_options) + count = 0 + if @pre_count + count = @pre_count + elsif !@count && @do_count + page_options[:count] = :count + r = Goo::SPARQL::Queries.model_load(page_options) + if r.is_a? Numeric + count = r.to_i + end + elsif @count + count = @count + end + page_options.delete :count + count + end + + def redis_indexed_ids + raise ArgumentError, "Redis is not configured" unless Goo.redis_client + rclient = Goo.redis_client + cache_key = cache_key_for_index(@index_key) + raise ArgumentError, "Index not found" unless rclient.exists(cache_key) + if @page_i + if !@count + @count = rclient.llen(cache_key) + end + rstart = (@page_i -1) * @page_size + rstop = (rstart + @page_size) -1 + ids = rclient.lrange(cache_key,rstart,rstop) + else + ids = rclient.lrange(cache_key,0,-1) + end + ids = ids.map { |i| RDF::URI.new(i) } + end + end + end +end diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb index 4e85d52e..e699a4e4 100644 --- a/lib/goo/sparql/queries.rb +++ b/lib/goo/sparql/queries.rb @@ -8,10 +8,10 @@ module Queries def self.duplicate_attribute_value?(model,attr,store=:main) value = model.instance_variable_get("@#{attr}") if !value.instance_of? Array - so = Goo.sparql_query_client(store).ask.from(model.graph). - whether([:id, model.class.attribute_uri(attr), value]). + so = Goo.sparql_query_client(store).select(:id).from(model.graph). + where([:id, model.class.attribute_uri(attr), value]). filter("?id != #{model.id.to_ntriples}") - return so.true? + return !so.solutions.empty? else #not yet support for unique arrays end @@ -45,11 +45,12 @@ def self.graph_predicates(*graphs) def self.model_exist(model,id=nil,store=:main) id = id || model.id - so = Goo.sparql_query_client(store).ask.from(model.graph). - whether([id, RDF.type, model.class.uri_type(model.collection)]) - return so.true? - end + so = Goo.sparql_query_client(store).select(:id).from(model.graph). + where([:id, RDF.type, model.class.uri_type(model.collection)]) + .filter("?id = #{id.to_ntriples}") + return !so.solutions.empty? + end def self.model_load(*options) Goo::SPARQL::Loader.model_load(*options) diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb index 7ef72193..a6e03a58 100644 --- a/lib/goo/sparql/query_builder.rb +++ b/lib/goo/sparql/query_builder.rb @@ -14,32 +14,31 @@ def initialize(options) @unions = options[:unions] || [] @aggregate = options[:aggregate] @collection = options[:collection] - @model_query_options = options[:query_options] @enable_rules = options[:rules] @order_by = options[:order_by] - + @internal_variables_map = {} + @equivalent_predicates = options[:equivalent_predicates] + @properties_to_include = options[:properties_to_include] @query = get_client end - def build_select_query(ids, variables, graphs, patterns, - query_options, properties_to_include) - - internal_variables = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions) + def build_query(ids, variables, graphs, patterns) + query_options = {} - aggregate_projections, aggregate_vars, - variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, - @klass, @unions, variables, internal_variables) + expand_equivalent_predicates(@properties_to_include, @equivalent_predicates) - @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables) + properties_to_include = @properties_to_include + patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions) variables, patterns = add_some_type_to_id(patterns, query_options, variables) - - query_filter_str, patterns, optional_patterns = - filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters) + aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables) + query_filter_str, patterns, optional_patterns, filter_variables = + filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters) + @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs) variables = [] if @count variables.delete :some_type - select_distinct(variables, aggregate_projections) + select_distinct(variables, aggregate_projections, filter_variables) .from(graphs) .where(patterns) .union_bind_in_where(properties_to_include) @@ -52,45 +51,50 @@ def build_select_query(ids, variables, graphs, patterns, @query.filter(filter) end - @query.union(*@unions) unless @unions.empty? + Array(@unions).each do |union| + @query.union(*union) + end ids_filter(ids) if ids - order_by if @order_by # TODO test if work + + + @query.order_by(*order_by_string) if @order_by + put_query_aggregate_vars(aggregate_vars) if aggregate_vars count if @count paginate if @page - ## TODO see usage of rules and query_options - query_options.merge!(@model_query_options) if @model_query_options - query_options[:rules] = [:NONE] unless @enable_rules - query_options = nil if query_options.empty? - if query_options - query_options[:rules] = query_options[:rules]&.map { |x| x.to_s }.join('+') - else - query_options = { rules: ['NONE'] } - end - @query.options[:query_options] = query_options [@query, aggregate_projections] end def union_bind_in_where(properties) binding_as = [] - properties.each do |property_attr, property| - predicates = [property[:uri]] + (property[:equivalents] || []) - options = { - binds: [{ value: property_attr, as: :attributeProperty }] - } - subject = property[:subject] || :id - predicates.uniq.each do |predicate_uri| - pattern = if property[:is_inverse] - [:attributeObject, predicate_uri, subject] - else - [subject, predicate_uri, :attributeObject] - end - binding_as << [[pattern], options] + if Goo.backend_4s? || Goo.backend_gb? + properties.each do |property_attr, property| + predicates = [property[:uri]] + (property[:equivalents] || []) + options = { + binds: [{ value: property_attr, as: :attributeProperty }] + } + subject = property[:subject] || :id + predicates.uniq.each do |predicate_uri| + pattern = if property[:is_inverse] + [:attributeObject, predicate_uri, subject] + else + [subject, predicate_uri, :attributeObject] + end + binding_as << [[pattern], options] + end end + + else + direct_predicate, inverse_predicate = include_properties + direct_filter = direct_predicate.empty? ? [] : [{ values: direct_predicate, predicate: :attributeProperty }] + inverse_filter = inverse_predicate.empty? ? [] : [{ values: inverse_predicate, predicate: :attributeProperty }] + binding_as << [[[:id, :attributeProperty, :attributeObject]], { filters: direct_filter}] unless direct_filter.empty? + binding_as << [[[:inverseAttributeObject, :attributeProperty, :id]], { filters: inverse_filter}] unless inverse_filter.empty? end + @query.optional_union_with_bind_as(*binding_as) unless binding_as.empty? self end @@ -117,10 +121,17 @@ def put_query_aggregate_vars(aggregate_vars) self end - def order_by - order_by_str = @order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" } - @query.order_by(*order_by_str) - self + def order_by_string + order_variables = [] + order_str = @order_by&.map do |attr, order| + if order.is_a?(Hash) + sub_attr, order = order.first + attr = @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last + end + order_variables << attr + "#{order.to_s.upcase}(?#{attr})" + end + order_str end def from(graphs) @@ -135,10 +146,11 @@ def from(graphs) self end - def select_distinct(variables, aggregate_projections) - + def select_distinct(variables, aggregate_patterns, filter_variables) + variables << :inverseAttributeObject if inverse_predicate? select_vars = variables.dup - reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections + reject_aggregations_from_vars(select_vars, aggregate_patterns) if aggregate_patterns + select_vars = (select_vars + filter_variables).uniq if @page && Goo.backend_4s? # Fix for 4store pagination with a filter @query = @query.select(*select_vars).distinct(true) self end @@ -147,7 +159,7 @@ def ids_filter(ids) filter_id = [] ids.each do |id| - filter_id << "?id = #{id.to_ntriples.to_s}" + filter_id << "?id = #{id.to_ntriples.to_s.gsub(' ', '%20').gsub("\\u0020", '%20')}" end filter_id_str = filter_id.join ' || ' @query.filter filter_id_str @@ -156,25 +168,37 @@ def ids_filter(ids) private + def include_properties + direct_predicates = @properties_to_include.select { |_, property| !property[:is_inverse] }.map { |_, property| [property[:uri]] + (property[:equivalents] || []) }.flatten + inverse_predicates = @properties_to_include.select { |_, property| property[:is_inverse] }.map { |_, property| [property[:uri]] + (property[:equivalents] || []) }.flatten + [direct_predicates, inverse_predicates] + end + + def inverse_predicate? + @properties_to_include.any? { |_, property| property[:is_inverse] } + end + def patterns_for_match(klass, attr, value, graphs, patterns, unions, internal_variables, subject = :id, in_union = false, in_aggregate = false, query_options = {}, collection = nil) + new_internal_var = value if value.respond_to?(:each) || value.instance_of?(Symbol) next_pattern = value.instance_of?(Array) ? value.first : value #for filters next_pattern = { next_pattern => [] } if next_pattern.instance_of?(Symbol) - value = "internal_join_var_#{internal_variables.length}".to_sym + new_internal_var = "internal_join_var_#{internal_variables.length}".to_sym if in_aggregate - value = "#{attr}_agg_#{in_aggregate}".to_sym + new_internal_var = "#{attr}_agg_#{in_aggregate}".to_sym end - internal_variables << value + internal_variables << new_internal_var + @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value} end - add_rules(attr, klass, query_options) + graph, pattern = - query_pattern(klass, attr, value: value, subject: subject, collection: collection) + query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection) if pattern if !in_union patterns << pattern @@ -187,7 +211,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions, range = klass.range(attr) next_pattern.each do |next_attr, next_value| patterns_for_match(range, next_attr, next_value, graphs, - patterns, unions, internal_variables, subject = value, + patterns, unions, internal_variables, subject = new_internal_var, in_union, in_aggregate, collection = collection) end end @@ -210,7 +234,7 @@ def walk_pattern(klass, match_patterns, graphs, patterns, unions, end end - def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, internal_variables) + def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables) # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT # https://github.com/ncbo/goo/issues/106 # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample @@ -241,8 +265,6 @@ def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, end def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions) - internal_variables = [] - if graph_match #make it deterministic - for caching graph_match_iteration = Goo::Base::PatternIteration.new(graph_match) @@ -250,28 +272,58 @@ def graph_match(collection, graph_match, graphs, klass, patterns, query_options, internal_variables, in_aggregate = false, query_options, collection) graphs.uniq! end - internal_variables + patterns end def get_client Goo.sparql_query_client(@store) end - def init_order_by(count, klass, order_by, optional_patterns, variables) + def init_order_by(count, klass, order_by, optional_patterns, variables, patterns, query_options, graphs) order_by = nil if count if order_by order_by = order_by.first #simple ordering ... needs to use pattern inspection order_by.each do |attr, direction| - quad = query_pattern(klass, attr) - optional_patterns << quad[1] + + if direction.is_a?(Hash) + # TODO this part can be improved/refactored, the complexity was added because order by don't work + # if the pattern is in the mandatory ones (variable `patterns`) + # and optional (variable `optional_patterns`) at the same time + sub_attr, direction = direction.first + graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]})) + old_internal = internal_variables.dup + old_patterns = optional_patterns.dup + + walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection) + new_variables = (internal_variables - old_internal) + internal_variables.delete(new_variables) + new_patterns = optional_patterns - old_patterns + already_existent_pattern = patterns.select{|x| x[1].eql?(new_patterns.last[1])}.first + + if already_existent_pattern + already_existent_variable = already_existent_pattern[2] + optional_patterns = old_patterns + key = @internal_variables_map.select{|key, value| key.eql?(new_variables.last)}.keys.first + @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key + + #variables << already_existent_variable + else + #variables << new_variables.last + end + + else + quad = query_pattern(klass, attr) + optional_patterns << quad[1] + #variables << attr + end + #patterns << quad[1] #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select #variables << attr unless variables.include?(attr) end - variables = %i[id attributeProperty attributeObject] end - [order_by, variables, optional_patterns] + [order_by, variables, optional_patterns, patterns] end def sparql_op_string(op) @@ -310,38 +362,48 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs, end filter_var = inspected_patterns[filter_pattern_match] - if !filter_operation.value.instance_of?(Goo::Filter) - if filter_operation.operator == :unbound || filter_operation.operator == :bound - if filter_operation.operator == :unbound - filter_operations << "!BOUND(?#{filter_var.to_s})" - else - filter_operations << "BOUND(?#{filter_var.to_s})" - end + if filter_operation.value.instance_of?(Goo::Filter) + filter_operations << "#{sparql_op_string(filter_operation.operator)}" + query_filter_sparql(klass, filter_operation.value, filter_patterns, + filter_graphs, filter_operations, + internal_variables, inspected_patterns, collection) + else + case filter_operation.operator + when :unbound + filter_operations << "!BOUND(?#{filter_var.to_s})" + return :optional + + when :bound + filter_operations << "BOUND(?#{filter_var.to_s})" return :optional + when :regex + if filter_operation.value.is_a?(String) + filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\", \"i\")" + end + else value = RDF::Literal.new(filter_operation.value) if filter_operation.value.is_a? String - value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string) + value = RDF::Literal.new(filter_operation.value) + filter_var = "str(?#{filter_var})" + else + filter_var = "?#{filter_var}" end filter_operations << ( - "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " + - " #{value.to_ntriples}") + "#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " + + " #{value.to_ntriples.to_s.gsub("\\u0020", "%20")}") end - else - filter_operations << "#{sparql_op_string(filter_operation.operator)}" - query_filter_sparql(klass, filter_operation.value, filter_patterns, - filter_graphs, filter_operations, - internal_variables, inspected_patterns, collection) + end end end - def filter_query_strings(collection, graphs, internal_variables, klass, + def filter_query_strings(collection, graphs, klass, optional_patterns, patterns, query_filters) query_filter_str = [] - filter_graphs = [] + filter_variables = [] inspected_patterns = {} query_filters&.each do |query_filter| filter_operations = [] @@ -358,9 +420,9 @@ def filter_query_strings(collection, graphs, internal_variables, klass, patterns.concat(filter_patterns) end end + #filter_variables << inspected_patterns.values.last end - - [query_filter_str, patterns, optional_patterns, internal_variables] + [query_filter_str, patterns, optional_patterns, filter_variables] end def reject_aggregations_from_vars(variables, aggregate_projections) @@ -376,6 +438,19 @@ def add_some_type_to_id(patterns, query_options, variables) [variables, patterns] end + def internal_variables + @internal_variables_map.keys + end + + def expand_equivalent_predicates(query_properties, eq_p) + + return unless eq_p && !eq_p.empty? + + query_properties&.each do |_, property| + property_uri = property[:uri] + property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s) + end + end end end end diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb index 3e6954f3..5848d0f7 100644 --- a/lib/goo/sparql/solutions_mapper.rb +++ b/lib/goo/sparql/solutions_mapper.rb @@ -1,42 +1,49 @@ module Goo module SPARQL class SolutionMapper - BNODES_TUPLES = Struct.new(:id, :attribute) - def initialize(aggregate_projections, bnode_extraction, embed_struct, - incl_embed, klass_struct, models_by_id, - properties_to_include, unmapped, variables,ids, options) + def initialize(aggregate_projections, bnode_extraction, embed_struct,incl_embed, klass_struct, models_by_id, variables, options) @aggregate_projections = aggregate_projections @bnode_extraction = bnode_extraction @embed_struct = embed_struct @incl_embed = incl_embed + @incl = options[:include] @klass_struct = klass_struct @models_by_id = models_by_id - @properties_to_include = properties_to_include - @unmapped = unmapped + @properties_to_include = options[:properties_to_include] + @unmapped = options[:include] && options[:include].first.eql?(:unmapped) @variables = variables - @ids = ids - @klass = options[:klass] + @ids = models_by_id.keys @klass = options[:klass] @read_only = options[:read_only] - @incl = options[:include] @count = options[:count] @collection = options[:collection] + @options = options end def map_each_solutions(select) - - found = Set.new objects_new = {} - var_set_hash = {} list_attributes = Set.new(@klass.attributes(:list)) - all_attributes = Set.new(@klass.attributes(:all)) + @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang], unmapped: @unmapped, + list_attributes: list_attributes) + if @options[:page] + # for using prefixes before queries + # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch) + # that enables implicit internal ordering. The patch requires the prefix below + select.prefix('franzOption_imposeImplicitBasicOrdering: ') + # mdorf, 1/24/2024, AllegroGraph 8 introduced a new feature that allows caching OFFSET/LIMIT queries + select.prefix('franzOption_allowCachingResults: ') + end + + select.options[:bypass_cache] = @options[:bypass_cache] if @options.has_key?(:bypass_cache) + select.each_solution do |sol| + next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type] return sol[:count_var].object if @count @@ -60,53 +67,74 @@ def map_each_solutions(select) next end - v = sol[:attributeProperty].to_s.to_sym + predicates = find_predicate(sol[:attributeProperty], inverse: !sol[:inverseAttributeObject].nil?) + next if predicates.empty? - next if v.nil? || !all_attributes.include?(v) + object = if sol[:attributeObject] + sol[:attributeObject] + elsif sol[:inverseAttributeObject] + sol[:inverseAttributeObject] + end - object = sol[:attributeObject] - #bnodes - if bnode_id?(object, v) - objects_new = bnode_id_tuple(id, object, objects_new, v) - next + predicates.each do |predicate| + # bnodes + if bnode_id?(object, predicate) + objects_new = bnode_id_tuple(id, object, objects_new, predicate) + next + end + + objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate) + add_object_to_model(id, objects, object, predicate) end - object, objects_new = get_value_object(id, objects_new, object, list_attributes, v) - add_object_to_model(id, object, v, var_set_hash) end + # for this moment we are not going to enrich models , maybe we will use it if the results are empty + @lang_filter.fill_models_with_all_languages(@models_by_id) - init_unloaded_attributes(found, list_attributes) + init_unloaded_attributes(list_attributes) return @models_by_id if @bnode_extraction model_set_collection_attributes(@models_by_id, objects_new) - #remove from models_by_id elements that were not touched - @models_by_id.select! { |k, m| found.include?(k) } + # remove from models_by_id elements that were not touched + @models_by_id.select! { |k, _m| found.include?(k) } models_set_all_persistent(@models_by_id) unless @read_only - #next level of embed attributes + # next level of embed attributes include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty? - #bnodes - blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? } + # bnodes + blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? } include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty? models_unmapped_to_array(@models_by_id) if @unmapped + @models_by_id end private - def init_unloaded_attributes(found, list_attributes) - return if @incl.nil? + def find_predicate(predicate, unmapped: false, inverse: false) + if Goo.backend_4s? || Goo.backend_gb? + return [] if predicate.nil? || unmapped && @properties_to_include[predicate].nil? + predicate = predicate.to_s.to_sym + else + predicate = @properties_to_include.select { |x, v| v[:uri].to_s.eql?(predicate.to_s) || v[:equivalents]&.any? { |e| e.to_s.eql?(predicate.to_s) } } + return [] if predicate.empty? + + predicate = predicate.select{|x, y| y[:is_inverse]&.eql?(inverse)}.keys + end + Array(predicate) + end + def init_unloaded_attributes(list_attributes) + return if @incl.nil? || @incl.empty? # Here we are setting to nil all attributes that have been included but not found in the triplestore - found.uniq.each do |model_id| - m = @models_by_id[model_id] + @models_by_id.each do |id, m| @incl.each do |attr_to_incl| is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl) next if attr_to_incl.to_s.eql?('unmapped') || is_handler @@ -130,60 +158,44 @@ def init_unloaded_attributes(found, list_attributes) def get_value_object(id, objects_new, object, list_attributes, predicate) object = object.object if object && !(object.is_a? RDF::URI) range_for_v = @klass.range(predicate) - #binding.pry if v.eql?(:enrolled) - #dependent model creation + if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil? if objects_new.include?(object) object = objects_new[object] - elsif !range_for_v.inmutable? + else pre_val = get_preload_value(id, object, predicate) object, objects_new = if !@read_only preloaded_or_new_object(object, objects_new, pre_val, predicate) else - #depedent read only + # depedent read only preloaded_or_new_struct(object, objects_new, pre_val, predicate) end - else - object = range_for_v.find(object).first end end if list_attributes.include?(predicate) - # To handle attr that are lists - pre = if @klass_struct - @models_by_id[id][predicate] - else - @models_by_id[id].instance_variable_get("@#{predicate}") - end - if object.nil? && pre.nil? - object = [] - elsif object.nil? && !pre.nil? - object = pre - elsif object - object = !pre ? [object] : (pre.dup << object) + pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}") + + if object.nil? + object = pre.nil? ? [] : pre + else + object = pre.nil? ? [object] : (Array(pre).dup << object) object.uniq! end + end - [object,objects_new] + [object, objects_new] end - def add_object_to_model(id, object, predicate, var_set_hash) + def add_object_to_model(id, objects, current_obj, predicate) + if @models_by_id[id].respond_to?(:klass) - @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil? + @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil? elsif !@models_by_id[id].class.handler?(predicate) && - !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) && + !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) && predicate != :id - # if multiple language values are included for a given property, set the - # corresponding model attribute to the English language value - NCBO-1662 - if object.is_a?(RDF::Literal) - key = "#{predicate}#__#{id}" - @models_by_id[id].send("#{predicate}=", object, on_load: true) unless var_set_hash[key] - lang = object.language - var_set_hash[key] = true if %i[EN en].include?(lang) - else - @models_by_id[id].send("#{predicate}=", object, on_load: true) - end + @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj) end end @@ -191,7 +203,7 @@ def get_preload_value(id, object, predicate) pre_val = nil if predicate_preloaded?(id, predicate) pre_val = preloaded_value(id, predicate) - pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array) + pre_val = pre_val.select { |x| x.respond_to?(:id) && (x.id == object) }.first if pre_val.is_a?(Array) end pre_val end @@ -213,6 +225,7 @@ def preloaded_or_new_struct(object, objects_new, pre_val, predicate) def preloaded_value(id, predicate) if !@read_only @models_by_id[id].instance_variable_get("@#{predicate}") + else @models_by_id[id][predicate] end @@ -229,9 +242,7 @@ def bnode_id?(object, predicate) def bnode_id_tuple(id, object, objects_new, predicate) range = @klass.range(predicate) - if range.respond_to?(:new) - objects_new[object] = BNODES_TUPLES.new(id, predicate) - end + objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new) objects_new end @@ -245,21 +256,13 @@ def create_model(id) @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id) end - def model_set_unmapped(id, predicate, value) - - if @models_by_id[id].respond_to? :klass #struct - @models_by_id[id][:unmapped] ||= {} - (@models_by_id[id][:unmapped][predicate] ||= []) << value - else - @models_by_id[id].unmapped_set(predicate, value) - end - end def create_struct(bnode_extraction, models_by_id, sol, variables) list_attributes = Set.new(@klass.attributes(:list)) struct = @klass.range(bnode_extraction).new variables.each do |v| next if v == :id + svalue = sol[v] struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object end @@ -280,70 +283,78 @@ def create_class_model(id, klass, klass_struct) end def models_unmapped_to_array(models_by_id) - models_by_id.each do |idm, m| - m.unmmaped_to_array + models_by_id.each do |_idm, m| + @lang_filter.models_unmapped_to_array(m) end end + + def is_multiple_langs? + return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL) + false + end + def include_bnodes(bnodes, models_by_id) - #group by attribute - attrs = bnodes.map { |x, y| y.attribute }.uniq + # group by attribute + attrs = bnodes.map { |_x, y| y.attribute }.uniq attrs.each do |attr| struct = @klass.range(attr) - #bnodes that are in a range of goo ground models - #for example parents and children in LD class models - #we skip this cases for the moment + # bnodes that are in a range of goo ground models + # for example parents and children in LD class models + # we skip this cases for the moment next if struct.respond_to?(:model_name) bnode_attrs = struct.new.to_h.keys - ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id } - @klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values) - .in(@collection) - .include(bnode: { attr => bnode_attrs }).all + ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id } + @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values) + .in(@collection) + .include(bnode: { attr => bnode_attrs }).all end end def include_embed_attributes(incl_embed, objects_new) incl_embed.each do |attr, next_attrs| - #anything to join ? + # anything to join ? attr_range = @klass.range(attr) next if attr_range.nil? - range_objs = objects_new.select { |id, obj| + + range_objs = objects_new.select do |_id, obj| obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range) - }.values - unless range_objs.empty? - range_objs.uniq! - query = attr_range.where().models(range_objs).in(@collection).include(*next_attrs) - query = query.read_only if @read_only - query.all - end + end.values + next if range_objs.empty? + + range_objs.uniq! + query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs) + query = query.read_only if @read_only + query.all end end def models_set_all_persistent(models_by_id) return unless @ids - models_by_id.each do |k, m| + + models_by_id.each do |_k, m| m.persistent = true end end def model_set_collection_attributes(models_by_id, objects_new) collection_value = get_collection_value - if collection_value - collection_attribute = @klass.collection_opts - models_by_id.each do |id, m| - m.send("#{collection_attribute}=", collection_value) - end - objects_new.each do |id, obj_new| - if obj_new.respond_to?(:klass) - collection_attribute = obj_new[:klass].collection_opts - obj_new[collection_attribute] = collection_value - elsif obj_new.class.respond_to?(:collection_opts) && - obj_new.class.collection_opts.instance_of?(Symbol) - collection_attribute = obj_new.class.collection_opts - obj_new.send("#{collection_attribute}=", collection_value) - end + return unless collection_value + + collection_attribute = @klass.collection_opts + models_by_id.each do |_id, m| + m.send("#{collection_attribute}=", collection_value) + end + objects_new.each do |_id, obj_new| + if obj_new.respond_to?(:klass) + collection_attribute = obj_new[:klass].collection_opts + obj_new[collection_attribute] = collection_value + elsif obj_new.class.respond_to?(:collection_opts) && + obj_new.class.collection_opts.instance_of?(Symbol) + collection_attribute = obj_new.class.collection_opts + obj_new.send("#{collection_attribute}=", collection_value) end end end @@ -351,36 +362,12 @@ def model_set_collection_attributes(models_by_id, objects_new) def get_collection_value collection_value = nil if @klass.collection_opts.instance_of?(Symbol) - if @collection.is_a?(Array) && (@collection.length == 1) - collection_value = @collection.first - end - if @collection.respond_to? :id - collection_value = @collection - end + collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1) + collection_value = @collection if @collection.respond_to? :id end collection_value end - def model_map_attributes_values(id, var_set_hash, models_by_id, object, sol, v) - if models_by_id[id].respond_to?(:klass) - models_by_id[id][v] = object if models_by_id[id][v].nil? - else - model_attribute_val = models_by_id[id].instance_variable_get("@#{v.to_s}") - if (!models_by_id[id].class.handler?(v) || model_attribute_val.nil?) && v != :id - # if multiple language values are included for a given property, set the - # corresponding model attribute to the English language value - NCBO-1662 - if sol[v].is_a?(RDF::Literal) - key = "#{v}#__#{id.to_s}" - models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key] - lang = sol[v].language - var_set_hash[key] = true if %i[EN en EN en EN en].include?(lang) - else - models_by_id[id].send("#{v}=", object, on_load: true) - end - end - end - end - def object_to_array(id, klass_struct, models_by_id, object, predicate) pre = if klass_struct models_by_id[id][predicate] @@ -399,18 +386,15 @@ def object_to_array(id, klass_struct, models_by_id, object, predicate) end def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options) - read_only = options[:read_only] if object.is_a?(RDF::URI) && v != :id range_for_v = @klass.range(v) if range_for_v if objects_new.include?(object) object = objects_new[object] - elsif !range_for_v.inmutable? - pre_val = get_pre_val(id, models_by_id, object, v, read_only) - object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options) else - object = range_for_v.find(object).first + pre_val = get_pre_val(id, models_by_id, object, v) + object = get_object_from_range(pre_val, embed_struct, object, objects_new, v) end end end @@ -418,13 +402,12 @@ def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new end def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate) - range_for_v = @klass.range(predicate) if !@read_only object = pre_val || @klass.range_object(predicate, object) objects_new[object.id] = object else - #depedent read only + # depedent read only struct = pre_val || embed_struct[predicate].new struct.id = object struct.klass = range_for_v @@ -450,15 +433,18 @@ def get_pre_val(id, models_by_id, object, predicate) pre_val end - def add_unmapped_to_model(sol) - predicate = sol[:attributeProperty].to_s.to_sym - return unless @properties_to_include[predicate] - - id = sol[:id] - value = sol[:attributeObject] - - model_set_unmapped(id, @properties_to_include[predicate][:uri], value) + predicates = find_predicate(sol[:attributeProperty]) + predicates.each do |predicate| + if Goo.backend_4s? || Goo.backend_gb? + predicate = @properties_to_include[predicate][:uri] + else + predicate = sol[:attributeProperty] + end + id = sol[:id] + value = sol[:attributeObject] + @lang_filter.set_unmapped_value(@models_by_id[id], predicate, value) + end end def add_aggregations_to_model(sol) @@ -476,4 +462,3 @@ def add_aggregations_to_model(sol) end end end - diff --git a/lib/goo/sparql/sparql.rb b/lib/goo/sparql/sparql.rb index dfd3d0a6..d5315cde 100644 --- a/lib/goo/sparql/sparql.rb +++ b/lib/goo/sparql/sparql.rb @@ -1,9 +1,11 @@ require "sparql/client" require_relative "mixins/query_pattern" +require_relative "mixins/solution_lang_filter" require_relative "query_builder" require_relative "solutions_mapper" require_relative "client" require_relative "triples" require_relative "loader" require_relative "queries" +require_relative 'processor' diff --git a/lib/goo/sparql/triples.rb b/lib/goo/sparql/triples.rb index cb840df9..317d1d84 100644 --- a/lib/goo/sparql/triples.rb +++ b/lib/goo/sparql/triples.rb @@ -53,6 +53,8 @@ def self.model_update_triples(model) if model.previous_values graph_delete = RDF::Graph.new model.previous_values.each do |attr,value| + next unless model.modified_attributes.any?{|x| attr.eql?(x)} + predicate = model.class.attribute_uri(attr,model.collection) values = value.kind_of?(Array) ? value : [value] values.each do |v| @@ -67,16 +69,6 @@ def self.model_update_triples(model) unless model.persistent? graph_insert << [subject, RDF.type, model.class.uri_type(model.collection)] end - #set default values before saving - if not model.persistent? - model.class.attributes_with_defaults.each do |attr| - value = model.send("#{attr}") - if value.nil? - value = model.class.default(attr).call(model) - model.send("#{attr}=",value) - end - end - end model.modified_attributes.each do |attr| next if model.class.collection?(attr) diff --git a/lib/goo/utils/callbacks_utils.rb b/lib/goo/utils/callbacks_utils.rb new file mode 100644 index 00000000..b9e747ff --- /dev/null +++ b/lib/goo/utils/callbacks_utils.rb @@ -0,0 +1,22 @@ +module CallbackRunner + + def run_callbacks(inst, callbacks) + callbacks.each do |proc| + if instance_proc?(inst, proc) + call_proc(inst.method(proc)) + elsif proc.is_a?(Proc) + call_proc(proc) + end + end + end + + def instance_proc?(inst, opt) + opt && (opt.is_a?(Symbol) || opt.is_a?(String)) && inst.respond_to?(opt) + end + + def call_proc(proc) + proc.call + end + + +end \ No newline at end of file diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb index d326839b..dd10c35b 100644 --- a/lib/goo/validators/enforce.rb +++ b/lib/goo/validators/enforce.rb @@ -1,129 +1,140 @@ +require_relative '../utils/callbacks_utils' module Goo module Validators module Enforce - def self.enforce_by_attribute(model,attr) - return model.model_settings[:attributes][attr][:enforce] - end + class EnforceInstance + include CallbackRunner + attr_reader :errors_by_opt + def initialize + @errors_by_opt = {} + end + + def enforce(inst,attr,value) + enforce_opts = enforce_by_attribute(inst.class,attr) + return nil if enforce_opts.nil? or enforce_opts.length == 0 - def self.enforce_type_boolean(attr,value) - if value.kind_of? Array - if (value.select {|x| !((x.class == TrueClass) || (x.class == FalseClass))} ).length > 0 - return "All values in attribute `#{attr}` must be `Boolean`" + enforce_opts.each do |opt| + case opt + when :unique + check Goo::Validators::Unique, inst, attr, value, opt + when :no_list + validator = Goo::Validators::DataType.new(inst, attr, value, Array) + if validator.valid? && !value.nil? + add_error(opt, + "`#{attr}` is defined as non Array - it cannot hold multiple values") + end + when :existence + check Goo::Validators::Existence, inst, attr, value, opt + when :list, Array + check Goo::Validators::DataType, inst, attr, value, opt, Array + when :uri, RDF::URI + check Goo::Validators::DataType, inst, attr, value, opt, RDF::URI + when :url + check Goo::Validators::DataType, inst, attr, value, opt, :url + when :string, String + check Goo::Validators::DataType, inst, attr, value, opt, String + when :integer, Integer + check Goo::Validators::DataType, inst, attr, value, opt, Integer + when :boolean + check Goo::Validators::DataType, inst, attr, value, opt,:boolean + when :date_time, DateTime + check Goo::Validators::DataType, inst, attr, value, opt, DateTime + when :float, Float + check Goo::Validators::DataType, inst, attr, value, opt, Float + when :symmetric + check Goo::Validators::Symmetric, inst, attr, value, opt + when :email + check Goo::Validators::Email, inst, attr, value, opt + when :username + check Goo::Validators::Username, inst, attr, value, opt + when /^distinct_of_/ + check Goo::Validators::DistinctOf, inst, attr, value, opt, opt + when /^superior_equal_to_/ + check Goo::Validators::SuperiorEqualTo, inst, attr, value, opt, opt + when /^inverse_of_/ + check Goo::Validators::InverseOf, inst, attr, value, opt, opt + when Proc + call_proc(opt, inst, attr) + when /^max_/, /^min_/ + type = opt.to_s.index("max_") ? :max : :min + check Goo::Validators::ValueRange, inst, attr, value, type, opt.to_s + when /^safe_text/ + check Goo::Validators::SafeText, inst, attr, value, opt, opt.to_s + else + if object_type?(opt) + check_object_type inst, attr, value, opt + elsif instance_proc?(inst, opt) + call_proc(inst.method(opt), inst, attr) + end + end end - else - if !((value.class == TrueClass) || (value.class == FalseClass)) - return "Attribute `#{attr}` value `#{value}` must be a `Boolean`" + + errors_by_opt.length > 0 ? errors_by_opt : nil + end + + def enforce_callback(inst, attr) + callbacks = Array(inst.class.attribute_callbacks(attr)) + callbacks.each do |proc| + if instance_proc?(inst, proc) + call_proc(inst.method(proc), inst, attr) + elsif proc.is_a?(Proc) + call_proc(proc, inst, attr) + end end end - end - def self.enforce_type(attr,type,value) - if type == :boolean - return self.enforce_type_boolean(attr,value) + private + + def object_type(opt) + opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt) end - if value.kind_of? Array - if (value.select {|x| !(x.kind_of? type)} ).length > 0 - return "All values in attribute `#{attr}` must be `#{type.name}`" - end - else - if !(value.kind_of? type) - return "Attribute `#{attr}` value `#{value}` must be a `#{type.name}`" + + def object_type?(opt) + opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt) + end + + def check_object_type(inst, attr, value, opt) + model_range = object_type(opt) + if model_range && !value.nil? + check Goo::Validators::ObjectType, inst, attr, value, model_range.model_name, model_range end end - end - def self.enforce_range_length(type_range,attr,opt_s,value) - if !value.nil? && !(value.kind_of?(Array) || value.kind_of?(String)) - return "#{attr} value (#{value}) must be an Array or String - it has range length constraints" + def check(validator_class, inst, attr, value, opt, *options) + validator = validator_class.new(inst, attr, value, *options) + add_error(opt, validator.error) unless validator.valid? end - range = opt_s[4..opt_s.length].to_i - if type_range == :min - if !value.nil? && (value.length < range) - return "#{attr} value has length `#{value.length}` and the min length is `#{range}`" - end - else - if !value.nil? && (value.length > range) - return "#{attr} value has length `#{value.length}` and the max length is `#{range}`" + def enforce_by_attribute(model, attr) + model.model_settings[:attributes][attr][:enforce] + end + + def call_proc(proc,inst, attr) + # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"] + errors = proc.call(inst, attr) + + return unless !errors.nil? && errors.is_a?(Array) + + errors.each_slice(2) do |e| + next if e.nil? || e.compact.empty? + add_error(e[0].to_sym, e[1]) end end + + def add_error(opt, err) + return if err.nil? + @errors_by_opt[opt] = err + end end + def self.enforce(inst,attr,value) - enforce_opts = enforce_by_attribute(inst.class,attr) - return nil if enforce_opts.nil? or enforce_opts.length == 0 - errors_by_opt = {} - enforce_opts.each do |opt| - case opt - when :unique - unless value.nil? - dup = Goo::SPARQL::Queries.duplicate_attribute_value?(inst,attr) - if dup - add_error(opt, errors_by_opt, - "`#{attr}` must be unique. " + - "There are other model instances with the same attribute value `#{value}`.") - end - end - when :no_list - if value.kind_of? Array - add_error(opt, errors_by_opt, - "`#{attr}` is defined as non Array - it cannot hold multiple values") - end - when :existence - add_error(opt, errors_by_opt, "`#{attr}` value cannot be nil") if value.nil? - when :list, Array - if !value.nil? && !(value.kind_of? Array) - add_error(opt, errors_by_opt, "`#{attr}` value must be an Array") - end - when :uri, RDF::URI - add_error(opt, errors_by_opt, enforce_type(attr,RDF::URI,value)) unless value.nil? - when :string, String - add_error(opt, errors_by_opt, enforce_type(attr,String,value)) unless value.nil? - when :integer, Integer - add_error(opt, errors_by_opt, enforce_type(attr,Integer,value)) unless value.nil? - when :boolean - add_error(opt, errors_by_opt, enforce_type(attr,:boolean,value)) unless value.nil? - when :date_time, DateTime - add_error(opt, errors_by_opt, enforce_type(attr,DateTime,value)) unless value.nil? - when Proc - # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"] - errors = opt.call(inst, attr) - errors.each_slice(2) do |e| - next if e.nil? || e.compact.empty? - add_error(e[0].to_sym, errors_by_opt, e[1]) rescue binding.pry - end - else - model_range = opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt) - if model_range and !value.nil? - values = value.kind_of?(Array) ? value : [value] - values.each do |v| - if (!v.kind_of?(model_range)) && !(v.respond_to?(:klass) && v[:klass] == model_range) - add_error(model_range.model_name, errors_by_opt, - "`#{attr}` contains values that are not instance of `#{model_range.model_name}`") - else - if !v.respond_to?(:klass) && !v.persistent? - add_error(model_range.model_name, errors_by_opt, - "`#{attr}` contains non persistent models. It will not save.") - end - end - end - end - opt_s = opt.to_s - if opt_s.index("max_") == 0 - add_error(:max, errors_by_opt, enforce_range_length(:max,attr,opt_s,value)) unless value.nil? - end - if opt_s.index("min_") == 0 - add_error(:min, errors_by_opt, enforce_range_length(:min,attr,opt_s,value)) unless value.nil? - end - end - end - return errors_by_opt.length > 0 ? errors_by_opt : nil + EnforceInstance.new.enforce(inst,attr,value) end - def self.add_error(opt, h, err) - return if err.nil? - h[opt] = err + def self.enforce_callbacks(inst, attr) + EnforceInstance.new.enforce_callback(inst, attr) end end end diff --git a/lib/goo/validators/implementations/data_type.rb b/lib/goo/validators/implementations/data_type.rb new file mode 100644 index 00000000..14440050 --- /dev/null +++ b/lib/goo/validators/implementations/data_type.rb @@ -0,0 +1,80 @@ +require 'uri' + +module Goo + module Validators + class DataType < ValidatorBase + include Validator + MAX_URL_LENGTH = 2048 + + keys %i[list uri url string integer boolean date_time float] + + error_message ->(_obj) { + if @value.is_a?(Array) + "All values in attribute `#{@attr}` must be `#{@type}`" + else + "Attribute `#{@attr}` with the value `#{@value}` must be `#{@type}`" + end + } + + validity_check ->(_obj) { enforce_type(@type, @value) } + + def initialize(inst, attr, value, type) + super(inst, attr, value) + @type = type + end + + def enforce_type(type, value) + return true if value.nil? + return enforce_type_boolean(value) if type == :boolean + return enforce_type_uri(value) if [:uri, RDF::URI].include?(type) + return enforce_type_url(value) if type == :url + return value.is_a?(Array) if type == Array + return value.all? { |x| x.is_a?(type) } if value.is_a?(Array) + + value.is_a?(type) + end + + def enforce_type_uri(value) + return true if value.nil? + return value.all? { |x| uri?(x) } if value.is_a?(Array) + + uri?(value) + end + + def enforce_type_url(value) + return true if value.nil? + return value.all? { |x| url?(x) } if value.is_a?(Array) + + url?(value) + end + + def enforce_type_boolean(value) + if value.is_a?(Array) + value.all? { |x| boolean?(x) } + else + boolean?(value) + end + end + + private + + def boolean?(value) + value.instance_of?(TrueClass) || value.instance_of?(FalseClass) + end + + def uri?(value) + value.is_a?(RDF::URI) && value.valid? + end + + def url?(value) + s = value.to_s + return false if s.empty? || s.length > MAX_URL_LENGTH + + uri = URI.parse(s) + uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty? + rescue URI::InvalidURIError + false + end + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/distinct_of.rb b/lib/goo/validators/implementations/distinct_of.rb new file mode 100644 index 00000000..2e93313b --- /dev/null +++ b/lib/goo/validators/implementations/distinct_of.rb @@ -0,0 +1,34 @@ +module Goo + module Validators + class DistinctOf < ValidatorBase + include Validator + + key :distinct_of_ + + error_message ->(obj) { "`#{@attr}` must be distinct of `#{@property}`"} + + validity_check -> (obj) do + return true if self.class.empty_value?(@value) + + self.distinct?(@inst, @property, @value) + end + + def initialize(inst, attr, value, key) + super(inst, attr, value) + @property = self.class.property(key) + end + + + + def distinct?(inst, property, value) + target_values = self.class.attr_value(property, inst) + current_values = Array(value) + + !current_values.any?{ |x| self.find_any?(target_values, x)} + end + def find_any?(array, value) + array.any?{ |x| self.class.equivalent_value?(value, x)} + end + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/email.rb b/lib/goo/validators/implementations/email.rb new file mode 100644 index 00000000..c74a9c79 --- /dev/null +++ b/lib/goo/validators/implementations/email.rb @@ -0,0 +1,49 @@ +module Goo + module Validators + class Email < ValidatorBase + include Validator + # Matches reasonably valid emails (no double dots, no leading/trailing dots or hyphens, valid domain) + EMAIL_REGEXP = /\A + [a-z0-9!#$%&'*+\/=?^_`{|}~-]+ # local part + (?:\.[a-z0-9!#$%&'*+\/=?^_`{|}~-]+)* # dot-separated continuation in local + @ + (?:(?!-)[a-z0-9-]{1,63}(?(obj) { + if @value.kind_of? Array + return "All values in attribute `#{@attr}` must be valid email addresses" + else + return "Attribute `#{@attr}` with the value `#{@value}` must be a valid email address" + end + } + + private + + validity_check ->(obj) do + return true if @value.nil? + + values = @value.is_a?(Array) ? @value : [@value] + + values.all? do |email| + next false unless email.is_a?(String) + next false unless email.length.between?(MIN_LENGTH, MAX_LENGTH) + + local, domain = email.split('@', 2) + next false if local.nil? || domain.nil? + next false if local.length > LOCAL_PART_MAX || domain.length > DOMAIN_PART_MAX + + email.match?(EMAIL_REGEXP) + end + end + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/existence.rb b/lib/goo/validators/implementations/existence.rb new file mode 100644 index 00000000..fcf04d61 --- /dev/null +++ b/lib/goo/validators/implementations/existence.rb @@ -0,0 +1,17 @@ +module Goo + module Validators + class Existence < ValidatorBase + include Validator + + key :existence + + error_message ->(obj) { "`#{@value}` value cannot be nil"} + + validity_check -> (obj) do + not self.class.empty_value?(@value) + end + + + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/inverse_of.rb b/lib/goo/validators/implementations/inverse_of.rb new file mode 100644 index 00000000..60518af3 --- /dev/null +++ b/lib/goo/validators/implementations/inverse_of.rb @@ -0,0 +1,35 @@ +module Goo + module Validators + class InverseOf < ValidatorBase + include Validator + + key :inverse_of_ + + error_message ->(obj) { + "`#{@attr}` must be the inverse of ``#{@property}``" + } + + validity_check -> (obj) do + return true if self.class.empty_value?(@value) + + return Array(@value).select{|x| not inverse?(@property,x, @inst)}.empty? + end + + def initialize(inst, attr, value, key) + super(inst, attr, value) + @property = self.class.property(key) + end + + def inverse?(attr, value, source_object) + if self.class.respond_to?(attr, value) + target_values = self.class.attr_value(attr, value) + return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)} + end + + false + end + + + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/object_type.rb b/lib/goo/validators/implementations/object_type.rb new file mode 100644 index 00000000..3af97b41 --- /dev/null +++ b/lib/goo/validators/implementations/object_type.rb @@ -0,0 +1,46 @@ +module Goo + module Validators + class ObjectType < ValidatorBase + include Validator + + key :object_type + + error_message ->(obj) { + if @error.eql?(:persistence) + "`#{@attr}` contains non persistent models. It will not save." + else + "`#{@attr}` contains values that are not instance of `#{@model_range.model_name}`" + end + } + + validity_check -> (obj) do + values = Array(@value) + + unless values.select { |v| !self.is_a_model?(v, @model_range) }.empty? + @error = :no_range + return false + end + + unless values.select { |v| !self.persistent?(v) }.empty? + @error = :persistence + return false + end + + return true + end + + def initialize(inst, attr, value, model_range) + super(inst, attr, value) + @model_range = model_range + end + + def is_a_model?(value, model_range) + value.is_a?(model_range) || (value.respond_to?(:klass) && value[:klass] == model_range) + end + + def persistent?(value) + value.respond_to?(:klass) || value.persistent? + end + end + end +end diff --git a/lib/goo/validators/implementations/safe_text.rb b/lib/goo/validators/implementations/safe_text.rb new file mode 100644 index 00000000..28ae794e --- /dev/null +++ b/lib/goo/validators/implementations/safe_text.rb @@ -0,0 +1,54 @@ +module Goo + module Validators + class SafeText < ValidatorBase + include Validator + + SAFE_TEXT_REGEX = /\A[\p{L}\p{N} .,'\-@()&!$%\/\[\]:;*+=?#^_{}|~"]+\z/u.freeze + DISALLOWED_UNICODE = /[\u0000-\u001F\u007F\u00A0\u200B-\u200F\u2028-\u202F\u202E\u2066-\u2069]/u.freeze + + key :safe_text + + error_message ->(obj) { + # Truncate long string values for clarity + truncated_value = if @value.is_a?(String) && @value.length > 60 + "#{@value[0...57]}..." + else + @value + end + + prefix = if @value.is_a?(Array) + "All values in attribute `#{@attr}`" + else + "Attribute `#{@attr}` with the value `#{truncated_value}`" + end + + suffix = "must be safe text (no control or invisible Unicode characters, newlines, or disallowed punctuation)" + length_note = @max_length ? " and must not exceed #{@max_length} characters" : "" + + "#{prefix} #{suffix}#{length_note}" + } + + validity_check ->(obj) do + return true if @value.nil? + + Array(@value).all? do |val| + next false unless val.is_a?(String) + + length_ok = @max_length.nil? || val.length <= @max_length + length_ok && + val !~ /\R/ && + val =~ SAFE_TEXT_REGEX && + val !~ DISALLOWED_UNICODE + end + end + + def initialize(inst, attr, value, opt) + @max_length = nil + super(inst, attr, value) + match = opt.match(/_(\d+)$/) + @max_length = match[1].to_i if match && match[1] + end + + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb new file mode 100644 index 00000000..46676794 --- /dev/null +++ b/lib/goo/validators/implementations/superior_equal_to.rb @@ -0,0 +1,26 @@ +module Goo + module Validators + class SuperiorEqualTo < ValidatorBase + include Validator + + key :superior_equal_to_ + + error_message ->(obj) { + "`#{@attr}` must be superior or equal to `#{@property}`" + } + + validity_check -> (obj) do + target_values = self.class.attr_value(@property, @inst) + + return true if target_values.nil? || target_values.empty? + + return Array(@value).all? {|v| v.nil? || target_values.all?{|t_v| v >= t_v}} + end + + def initialize(inst, attr, value, key) + super(inst, attr, value) + @property = self.class.property(key) + end + end + end +end diff --git a/lib/goo/validators/implementations/symmetric.rb b/lib/goo/validators/implementations/symmetric.rb new file mode 100644 index 00000000..e9ceb3f4 --- /dev/null +++ b/lib/goo/validators/implementations/symmetric.rb @@ -0,0 +1,33 @@ +module Goo + module Validators + class Symmetric < ValidatorBase + include Validator + + key :symmetric + + error_message ->(obj) { + "`#{@attr}` must be symmetric" + } + + validity_check -> (obj) do + return true if self.class.empty_value?(@value) + + return Array(@value).select{|x| not symmetric?(@attr,x, @inst)}.empty? + end + + def symmetric?(attr, value, source_object) + if respond_to?(attr, value) + target_values = self.class.attr_value(attr, value) + return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)} + end + + return false + end + + def respond_to?(attr, object) + object && object.respond_to?(attr) + end + + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/unique.rb b/lib/goo/validators/implementations/unique.rb new file mode 100644 index 00000000..feb13a4b --- /dev/null +++ b/lib/goo/validators/implementations/unique.rb @@ -0,0 +1,20 @@ +module Goo + module Validators + class Unique < ValidatorBase + include Validator + + key :unique + + error_message ->(obj) { "`#{@attr}` must be unique. " + + "There are other model instances with the same attribute value `#{@value}`."} + + validity_check -> (obj) do + return true if @value.nil? + + !Goo::SPARQL::Queries.duplicate_attribute_value?(@inst,@attr) + end + + + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/username.rb b/lib/goo/validators/implementations/username.rb new file mode 100644 index 00000000..8e9e858c --- /dev/null +++ b/lib/goo/validators/implementations/username.rb @@ -0,0 +1,45 @@ +module Goo + module Validators + class Username < ValidatorBase + include Validator + + RESERVED_NAMES = %w[ + admin administrator root support system test guest owner user + webmaster help contact host mail ftp info api noc security + ].freeze + + USERNAME_LENGTH_RANGE = (3..32).freeze + + ASCII_ONLY_REGEX = /\A[\x20-\x7E]+\z/ + USERNAME_PATTERN = /\A[a-zA-Z](?!.*[._]{2})[a-zA-Z0-9._]{1,30}[a-zA-Z0-9]\z/ + INVISIBLE_CHARS = /[\u200B-\u200D\uFEFF]/ + + key :username + + error_message ->(obj) { + base_msg = if @value.is_a?(Array) + "All values in attribute `#{@attr}` must be valid usernames" + else + "Attribute `#{@attr}` with the value `#{@value}` must be a valid username" + end + "#{base_msg} (must be 3–32 chars, start with a letter, contain only ASCII letters/digits/dots/underscores, no invisible or reserved terms)" + } + + validity_check ->(obj) do + return true if @value.nil? + + Array(@value).all? do |username| + next false unless username.is_a?(String) + + username = username.strip + + USERNAME_LENGTH_RANGE.cover?(username.length) && + username.match?(ASCII_ONLY_REGEX) && + username.match?(USERNAME_PATTERN) && + !username.match?(INVISIBLE_CHARS) && + !RESERVED_NAMES.include?(username.downcase) + end + end + end + end +end \ No newline at end of file diff --git a/lib/goo/validators/implementations/value_range.rb b/lib/goo/validators/implementations/value_range.rb new file mode 100644 index 00000000..71440bcf --- /dev/null +++ b/lib/goo/validators/implementations/value_range.rb @@ -0,0 +1,49 @@ +module Goo + module Validators + class ValueRange < ValidatorBase + include Validator + + keys [:min_, :max_] + + error_message ->(obj) { + value = self.value_length(@value) + if @type == :min + "#{@attr} value has length `#{value}` and the min length is `#{@range}`" + else + "#{@attr} value has length `#{value}` and the max length is `#{@range}`" + end + } + + validity_check -> (obj) do + self.enforce_range_length(@type, @range, @value) + end + + def initialize(inst, attr, value, type) + super(inst, attr, value) + @type = type.index("max_") ? :max : :min + @range = self.range(type) + end + + def enforce_range_length(type_range, range, value) + return false if value.nil? + value_length = self.value_length(value) + + (type_range.eql?(:min) && (value_length >= range)) || (type_range.eql?(:max) && (value_length <= range)) + end + + def range(opt) + opt[4..opt.length].to_i + end + + def value_length(value) + return 0 if value.nil? + + if value.is_a?(String) || value.is_a?(Array) + value.length + else + value + end + end + end + end +end diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb new file mode 100644 index 00000000..2536f985 --- /dev/null +++ b/lib/goo/validators/validator.rb @@ -0,0 +1,113 @@ +module Goo + module Validators + + class ValidatorBase + + def initialize(inst, attr, value) + @inst = inst + @attr = attr + @value = value + end + + def valid? + self.instance_eval(&self.class.validator_settings[:check]) + end + + def error + message = self.class.validator_settings[:message] + if message.is_a? Proc + self.instance_eval(&message) + else + message + end + end + + end + + module Validator + + def self.included(base) + base.extend(ClassMethods) + end + + + module ClassMethods + + def key(id) + validator_settings[:id] = id + end + + def keys(ids) + key ids + end + + def validity_check(block) + validator_settings[:check] = block + end + + def error_message(message) + validator_settings[:message] = message + end + + def validator_settings + @validator_settings ||= {} + end + + def ids + Array(validator_settings[:id]) + end + + def property(key) + key[ids.first.size..key.size].to_sym + end + + def respond_to?(attr, object) + object && object.respond_to?(attr) + end + + + def equivalent_value?(object1, object2) + if object1.respond_to?(:id) && object2.respond_to?(:id) + object1.id.eql?(object2.id) + else + object2 == object1 + end + end + + def attr_value(attr, object) + object.bring attr if object.respond_to?(:bring?) && object.bring?(attr) + + Array(object.send(attr)) + end + + def empty_value?(value) + value.nil? || empty?(value) || empty_array?(value) + end + def empty?(value) + empty_string?(value) || empty_to_s?(value) + end + def empty_string?(string) + string.is_a?(String) && string.strip.empty? + end + + def empty_to_s?(object) + begin + object && object.to_s&.strip.empty? + rescue + return false + end + end + + def empty_array?(array) + array.is_a?(Array) && array && array.reject{|x| x.nil? || empty?(x)}.empty? + end + end + + + + + + end + end +end + diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000..83aa57a8 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +ruby = "2.7.8" diff --git a/rakelib/ontoportal_testkit.rake b/rakelib/ontoportal_testkit.rake new file mode 100644 index 00000000..39a3ec73 --- /dev/null +++ b/rakelib/ontoportal_testkit.rake @@ -0,0 +1 @@ +require "ontoportal/testkit/tasks" diff --git a/test/app/bioportal.rb b/test/app/bioportal.rb index 2cef2c2b..8c99df03 100644 --- a/test/app/bioportal.rb +++ b/test/app/bioportal.rb @@ -4,93 +4,92 @@ require_relative '../test_case' require_relative './query_profiler' -GooTest.configure_goo - module Test - module BioPortal + module BioPortal class Ontology < Goo::Base::Resource model :ontology, namespace: :bioportal, name_with: :acronym attribute :acronym, namespace: :omv, enforce: [:existence, :unique] attribute :name, namespace: :omv, enforce: [:existence] attribute :administeredBy, enforce: [:user, :existence] - end + end class User < Goo::Base::Resource model :user, name_with: :username attribute :username, enforce: [:existence, :unique] attribute :email, enforce: [:existence, :email] attribute :roles, enforce: [:list, :role, :existence] - attribute :created, enforce: [ DateTime ], + attribute :created, enforce: [DateTime], default: lambda { |record| DateTime.now } - attribute :notes, inverse: { on: :note, attribute: :owner} - end + attribute :notes, inverse: { on: :note, attribute: :owner } + end class Role < Goo::Base::Resource - model :role, :inmutable, name_with: :code + model :role, name_with: :code attribute :code, enforce: [:existence, :unique] attribute :users, inverse: { on: :user, attribute: :roles } end class Note < Goo::Base::Resource - model :note, name_with: lambda { |s| id_generator(s) } + model :note, name_with: lambda { |s| id_generator(s) } attribute :content, enforce: [:existence] attribute :ontology, enforce: [:existence, :ontology] attribute :owner, enforce: [:existence, :user] + def self.id_generator(inst) - return RDF::URI.new("http://example.org/note/" + inst.owner.username + "/" + Random.rand(1000000).to_s ) + return RDF::URI.new("http://example.org/note/" + inst.owner.username + "/" + Random.rand(1000000).to_s) end end def self.benchmark_data - Goo.sparql_query_client.reset_profiling - if false - 10.times do |i| - Role.new(code: "role#{i}").save - end - puts "Roles created" - 900.times do |i| - roles = [] - 2.times do |j| - roles << Role.find("role#{j}").first + Goo.sparql_query_client.reset_profiling + if false + 10.times do |i| + Role.new(code: "role#{i}").save + end + puts "Roles created" + 900.times do |i| + roles = [] + 2.times do |j| + roles << Role.find("role#{j}").first + end + u = User.new(username: "user#{i}name", email: "email#{i}@example.org", roles: roles) + u.save + puts "#{i} users created" + end + 400.times do |i| + ont = Ontology.new(acronym: "ontology #{i}", name: "ontology ontology ontology #{i}") + ont.administeredBy = User.find("user#{i % 75}name").first + ont.save + end + binding.pry + 1000.times do |i| + ont = Ontology.where(acronym: "ontology #{Random.rand(200)}").all.first + owner = User.where(username: "user#{i % 300}name").include(:username).all.first + n = Note.new(content: "content " * 60, owner: owner, ontology: ont) + n.save + puts "created note #{i}" + end + binding.pry + 2000.times do |i| + ont = Ontology.where(acronym: "ontology #{Random.rand(15)}").all.first + owner = User.where(username: "user#{i % 200}name").include(:username).all.first + n = Note.new(content: "content " * 60, owner: owner, ontology: ont) + n.save + puts "created note #{i}" + end + binding.pry + 800.times do |i| + ont = Ontology.where(acronym: "ontology #{Random.rand(6)}").all.first + owner = User.where(username: "user#{i % 200}name").include(:username).all.first + n = Note.new(content: "content " * 60, owner: owner, ontology: ont) + n.save + puts "created note #{i}" end - u = User.new(username: "user#{i}name", email: "email#{i}@example.org", roles: roles) - u.save - puts "#{i} users created" - end - 400.times do |i| - ont = Ontology.new(acronym: "ontology #{i}",name: "ontology ontology ontology #{i}") - ont.administeredBy = User.find("user#{i % 75}name").first - ont.save - end - binding.pry - 1000.times do |i| - ont = Ontology.where(acronym: "ontology #{Random.rand(200)}").all.first - owner = User.where(username: "user#{i % 300}name").include(:username).all.first - n = Note.new(content: "content " * 60, owner: owner, ontology: ont) - n.save - puts "created note #{i}" - end - binding.pry - 2000.times do |i| - ont = Ontology.where(acronym: "ontology #{Random.rand(15)}").all.first - owner = User.where(username: "user#{i % 200}name").include(:username).all.first - n = Note.new(content: "content " * 60, owner: owner, ontology: ont) - n.save - puts "created note #{i}" - end - binding.pry - 800.times do |i| - ont = Ontology.where(acronym: "ontology #{Random.rand(6)}").all.first - owner = User.where(username: "user#{i % 200}name").include(:username).all.first - n = Note.new(content: "content " * 60, owner: owner, ontology: ont) - n.save - puts "created note #{i}" end - end 500.times do |i| ont_id = 0 begin - ont_id = Random.rand(5)+180 + ont_id = Random.rand(5) + 180 end ont = Ontology.where(acronym: "ontology #{ont_id}").all.first owner = User.where(username: "user#{i % 200}name").include(:username).all.first @@ -100,12 +99,12 @@ def self.benchmark_data end end - def self.benchmark_naive_query + def self.benchmark_naive_query Goo.sparql_query_client.reset_profiling ont = Ontology.where.include(:acronym).all bench_result = [] ont.each do |ont| - qq =< . ?id ?username . @@ -130,12 +129,12 @@ def self.benchmark_naive_query users = {} roles = {} count_sol = 0 - res = client.query(qq) + res = client.query(qq) res.each do |sol| unless users.include?(sol[:id]) users[sol[:id]] = User.new - users[sol[:id]].username=sol[:username] - users[sol[:id]].email=sol[:email] + users[sol[:id]].username = sol[:username] + users[sol[:id]].email = sol[:email] end unless roles.include?(sol[:roles]) roles[sol[:roles]] = Role.new @@ -147,14 +146,14 @@ def self.benchmark_naive_query end count_sol = count_sol + 1 end - bench_result << [Time.now - start,notes.length, client.query_times.last, client.parse_times.last,count_sol ] + bench_result << [Time.now - start, notes.length, client.query_times.last, client.parse_times.last, count_sol] end bench_result.select! { |x| x[1] > 0 } bench_result.sort_by! { |x| x[1] } CSV.open("benchmark_naive.csv", "wb") do |csv| - csv << ["total", "notes", "qt", "pt","sol"] + csv << ["total", "notes", "qt", "pt", "sol"] bench_result.each do |b| - csv << b + csv << b end end end @@ -164,7 +163,7 @@ def self.benchmark_naive_fast ont = Ontology.where.include(:acronym).all bench_result = [] ont.each do |ont| - qq =< . ?note ?id . @@ -185,11 +184,11 @@ def self.benchmark_naive_fast users = {} roles = {} count_sol = 0 - res = client.query(qq) + res = client.query(qq) res.each do |sol| unless users.include?(sol[:id]) users[sol[:id]] = User.new - users[sol[:id]].username=sol[:username] + users[sol[:id]].username = sol[:username] end unless roles.include?(sol[:roles]) roles[sol[:roles]] = Role.new @@ -201,14 +200,14 @@ def self.benchmark_naive_fast end count_sol = count_sol + 1 end - bench_result << [Time.now - start,notes.length, client.query_times.last, client.parse_times.last,count_sol ] + bench_result << [Time.now - start, notes.length, client.query_times.last, client.parse_times.last, count_sol] end bench_result.select! { |x| x[1] > 0 } bench_result.sort_by! { |x| x[1] } CSV.open("benchmark_naive_fast.csv", "wb") do |csv| - csv << ["total", "notes", "qt", "pt","sol"] + csv << ["total", "notes", "qt", "pt", "sol"] bench_result.each do |b| - csv << b + csv << b end end end @@ -223,20 +222,20 @@ def self.benchmark_query_goo_fast start = Time.now notes = nil notes = Note.where(ontology: ont) - .include(:content) - .include(:owner) - .all + .include(:content) + .include(:owner) + .all num_queries = client.query_times.length - agg_parsing = client.parse_times.inject{|sum,x| sum + x } - agg_queries = client.query_times.inject{|sum,x| sum + x } - bench_result << [Time.now - start, notes.length,agg_queries,agg_parsing,num_queries ] + agg_parsing = client.parse_times.inject { |sum, x| sum + x } + agg_queries = client.query_times.inject { |sum, x| sum + x } + bench_result << [Time.now - start, notes.length, agg_queries, agg_parsing, num_queries] end bench_result.select! { |x| x[1] > 0 } bench_result.sort_by! { |x| x[1] } CSV.open("benchmark_goo_fast.csv", "wb") do |csv| csv << ["total", "notes", "agg_qt", "agg_qp", "queries"] bench_result.each do |b| - csv << b + csv << b end end end @@ -246,27 +245,26 @@ def self.benchmark_query_goo client.reset_profiling ont = Ontology.where.include(:acronym).all bench_result = [] - Role.load_inmutable_instances ont.each do |ont| client.reset_profiling start = Time.now notes = nil notes = Note.where(ontology: ont) - .include(:content) - .include(owner: [ :username, :email, roles: [:code]]) - .read_only - .all + .include(:content) + .include(owner: [:username, :email, roles: [:code]]) + .read_only + .all num_queries = client.query_times.length - agg_parsing = client.parse_times.inject{|sum,x| sum + x } - agg_queries = client.query_times.inject{|sum,x| sum + x } - bench_result << [Time.now - start, notes.length,agg_queries,agg_parsing,num_queries ] + agg_parsing = client.parse_times.inject { |sum, x| sum + x } + agg_queries = client.query_times.inject { |sum, x| sum + x } + bench_result << [Time.now - start, notes.length, agg_queries, agg_parsing, num_queries] end bench_result.select! { |x| x[1] > 0 } bench_result.sort_by! { |x| x[1] } CSV.open("benchmark_goo.csv", "wb") do |csv| csv << ["total", "notes", "agg_qt", "agg_qp", "queries"] bench_result.each do |b| - csv << b + csv << b end end end diff --git a/test/app/models.rb b/test/app/models.rb index 5aeb2a2e..876b70df 100644 --- a/test/app/models.rb +++ b/test/app/models.rb @@ -1,7 +1,5 @@ require_relative '../test_case' -GooTest.configure_goo - module Test module Models diff --git a/test/app/test_app.rb b/test/app/test_app.rb index 26a88d60..4d444ad4 100644 --- a/test/app/test_app.rb +++ b/test/app/test_app.rb @@ -1,6 +1,4 @@ require_relative '../test_case' require_relative 'bioportal' -GooTest.configure_goo - binding.pry diff --git a/test/console.rb b/test/console.rb index e64d4adf..39d19aa2 100644 --- a/test/console.rb +++ b/test/console.rb @@ -1,5 +1,4 @@ require_relative "../lib/goo.rb" require_relative "./test_case.rb" -GooTest.configure_goo binding.pry diff --git a/test/data/languages.nt b/test/data/languages.nt new file mode 100644 index 00000000..faf464f0 --- /dev/null +++ b/test/data/languages.nt @@ -0,0 +1,9 @@ + . + "John Doe"@en . + "Jean Dupont"@fr . + "Juan Pérez" . + . + "Paris"@en . + "Paris"@fr . + "París"@es . + "Berlin" . diff --git a/test/data/yaml_scheme_model_test.yml b/test/data/yaml_scheme_model_test.yml new file mode 100644 index 00000000..b184a117 --- /dev/null +++ b/test/data/yaml_scheme_model_test.yml @@ -0,0 +1,30 @@ +name: + label: 'Name' + description: 'Person name' + equivalents: ['test:name' , 'test2:name', 'test3:person_name'] + help: 'Put the person name as string' + example: 'John' +nationality: + label: 'Person nationality' + enforcedValues: {'fr': 'france', 'us': 'USA'} + + +test_string: + default: "Test String" + +test_integer: + default: 2 + +test_list: + default: + - item1 + - item2 + +test_float: + default: 3.14 + +test_uri: + default: https://example.com/term1 + +test_boolean: + default: false \ No newline at end of file diff --git a/test/models.rb b/test/models.rb index cd606eed..78ec1393 100644 --- a/test/models.rb +++ b/test/models.rb @@ -64,11 +64,26 @@ class Student < Goo::Base::Resource end module GooTestData + TRACKED_FIXTURE_MODELS = [Student, University, Program, Category, Address].freeze + + def self.safe_model_count(model) + model.where.include(model.attributes).all.length + rescue StandardError => e + warn "[GooTestData] count failed for #{model.name}: #{e.class}: #{e.message}" + -1 + end + + def self.log_fixture_counts(stage) + summary = TRACKED_FIXTURE_MODELS.map { |m| "#{m.name}=#{safe_model_count(m)}" }.join(", ") + puts "[GooTestData] #{stage}: #{summary}" + end + def self.create_test_case_data + log_fixture_counts("before create_test_case_data") addresses = {} - addresses["Stanford"] = [ Address.new(line1: "bla", line2: "foo", country: "US").save ] - addresses["Southampton"] = [ Address.new(line1: "bla", line2: "foo", country: "UK").save ] - addresses["UPM"] = [ Address.new(line1: "bla", line2: "foo", country: "SP").save ] + addresses["Stanford"] = [Address.where(line1: "bla", line2: "foo", country: "US").first || Address.new(line1: "bla", line2: "foo", country: "US").save] + addresses["Southampton"] = [Address.where(line1: "bla", line2: "foo", country: "UK").first || Address.new(line1: "bla", line2: "foo", country: "UK").save] + addresses["UPM"] = [Address.where(line1: "bla", line2: "foo", country: "SP").first || Address.new(line1: "bla", line2: "foo", country: "SP").save] ["Stanford", "Southampton", "UPM"].each do |uni_name| if University.find(uni_name).nil? University.new(name: uni_name, address: addresses[uni_name]).save @@ -79,7 +94,9 @@ def self.create_test_case_data end prg = Program.new(name: p, category: categories, credits: credits, university: University.find(uni_name).include(:name).first ) - binding.pry if !prg.valid? + unless prg.valid? + raise "Program fixture is invalid for university=#{uni_name.inspect}, program=#{p.inspect}. Errors: #{prg.errors.inspect}" + end prg.save if !prg.exist? end end @@ -96,12 +113,22 @@ def self.create_test_case_data programs << pr end st.enrolled= programs - st.save rescue binding.pry + begin + st.save + rescue StandardError => e + raise "#{e.class}: failed saving student fixture #{st_data[0].inspect}. #{e.message}" + end end + log_fixture_counts("after create_test_case_data") end def self.delete_test_case_data - objects = [Student, University, Program, Category, Address] + log_fixture_counts("before delete_test_case_data") + delete_all [Student, University, Program, Category, Address] + log_fixture_counts("after delete_test_case_data") + end + + def self.delete_all(objects) objects.each do |obj| obj.where.include(obj.attributes).each do |i| i.delete diff --git a/test/settings/test_hooks.rb b/test/settings/test_hooks.rb new file mode 100644 index 00000000..47d8fa0f --- /dev/null +++ b/test/settings/test_hooks.rb @@ -0,0 +1,50 @@ +require_relative '../test_case' + +class TestHookModel < Goo::Base::Resource + model :test_hook, name_with: lambda { |s| RDF::URI.new("http://example.org/test/#{rand(1000)}") } + after_save :update_count, :update_count_2 + after_destroy :decrease_count_2 + attribute :name, enforce: [:existence, :unique] + + attr_reader :count, :count2 + + def update_count + @count ||= 0 + @count += 1 + end + + def update_count_2 + @count2 ||= 0 + @count2 += 2 + end + + def decrease_count_2 + @count2 -= 2 + end + +end + +class TestHooksSetting < MiniTest::Unit::TestCase + + def test_model_hooks + TestHookModel.find("test").first&.delete + + model = TestHookModel.new(name: "test").save + + assert_equal 1, model.count + assert_equal 2, model.count2 + + model.name = "test2" + model.save + + assert_equal 2, model.count + assert_equal 4, model.count2 + + + model.delete + + assert_equal 2, model.count + assert_equal 2, model.count2 + + end +end diff --git a/test/solr/test_solr.rb b/test/solr/test_solr.rb new file mode 100644 index 00000000..6428bc8a --- /dev/null +++ b/test/solr/test_solr.rb @@ -0,0 +1,122 @@ +require_relative '../test_case' +require 'benchmark' + + +class TestSolr < MiniTest::Unit::TestCase + def self.before_suite + @@connector = SOLR::SolrConnector.new(Goo.search_conf, 'test') + @@connector.delete_collection('test') + @@connector.init + end + + def self.after_suite + @@connector.delete_collection('test') + end + + def test_add_collection + connector = @@connector + connector.create_collection('test2') + all_collections = connector.fetch_all_collections + assert_includes all_collections, 'test2' + end + + def test_delete_collection + connector = @@connector + test_add_collection + connector.delete_collection('test2') + + all_collections = connector.fetch_all_collections + refute_includes all_collections, 'test2' + end + + def test_schema_generator + connector = @@connector + + all_fields = connector.all_fields + + connector.schema_generator.fields_to_add.each do |f| + field = all_fields.select { |x| x["name"].eql?(f[:name]) }.first + refute_nil field + assert_equal field["type"], f[:type] + assert_equal field["indexed"], f[:indexed] + assert_equal field["stored"], f[:stored] + assert_equal field["multiValued"], f[:multiValued] + end + + copy_fields = connector.all_copy_fields + connector.schema_generator.copy_fields_to_add.each do |f| + field = copy_fields.select { |x| x["source"].eql?(f[:source]) }.first + refute_nil field + assert_equal field["source"], f[:source] + assert_includes f[:dest], field["dest"] + end + + dynamic_fields = connector.all_dynamic_fields + + connector.schema_generator.dynamic_fields_to_add.each do |f| + field = dynamic_fields.select { |x| x["name"].eql?(f[:name]) }.first + refute_nil field + assert_equal field["name"], f[:name] + assert_equal field["type"], f[:type] + assert_equal field["multiValued"], f[:multiValued] + assert_equal field["stored"], f[:stored] + end + + connector.clear_all_schema + connector.fetch_schema + all_fields = connector.all_fields + connector.schema_generator.fields_to_add.each do |f| + field = all_fields.select { |x| x["name"].eql?(f[:name]) }.first + assert_nil field + end + + copy_fields = connector.all_copy_fields + connector.schema_generator.copy_fields_to_add.each do |f| + field = copy_fields.select { |x| x["source"].eql?(f[:source]) }.first + assert_nil field + end + + dynamic_fields = connector.all_dynamic_fields + connector.schema_generator.dynamic_fields_to_add.each do |f| + field = dynamic_fields.select { |x| x["name"].eql?(f[:name]) }.first + assert_nil field + end + end + + def test_add_field + connector = @@connector + add_field('test', connector) + + + field = connector.fetch_all_fields.select { |f| f['name'] == 'test' }.first + + refute_nil field + assert_equal field['type'], 'string' + assert_equal field['indexed'], true + assert_equal field['stored'], true + assert_equal field['multiValued'], true + + connector.delete_field('test') + end + + def test_delete_field + connector = @@connector + + add_field('test', connector) + + connector.delete_field('test') + + field = connector.all_fields.select { |f| f['name'] == 'test' }.first + + assert_nil field + end + + private + + def add_field(name, connector) + if connector.fetch_field(name) + connector.delete_field(name) + end + connector.add_field(name, 'string', indexed: true, stored: true, multi_valued: true) + end +end diff --git a/test/test_basic_persistence.rb b/test/test_basic_persistence.rb index 5ca8ec2f..665a5d60 100644 --- a/test/test_basic_persistence.rb +++ b/test/test_basic_persistence.rb @@ -1,8 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - - module Dep class Ontology < Goo::Base::Resource model :ontology, name_with: :name @@ -264,10 +261,9 @@ def test_update end def test_update_array_values - #object should always return freezed arrays - #so that we detect the set - arr = ArrayValues.new(name: "x" , many: ["a","b"]) - assert (arr.valid?) + # Object should always return frozen arrays, so that we detect the set + arr = ArrayValues.new(name: "x" , many: ["a", "b"]) + assert arr.valid? arr.save assert arr.persistent? assert arr.exist? @@ -275,19 +271,18 @@ def test_update_array_values arr_from_backend = ArrayValues.find(arr.id).include(ArrayValues.attributes).first assert_equal ["a", "b"], arr_from_backend.many.sort - assert_raises RuntimeError do + assert_raises FrozenError do arr_from_backend.many << "c" end - arr_from_backend.many = ["A","B","C"] + arr_from_backend.many = ["A", "B", "C"] arr_from_backend.save arr_from_backend = ArrayValues.find(arr.id).include(ArrayValues.attributes).first - assert_equal ["A","B","C"], arr_from_backend.many.sort + assert_equal ["A", "B", "C"], arr_from_backend.many.sort arr_from_backend.delete assert !arr_from_backend.exist? - end def test_person_save diff --git a/test/test_cache.rb b/test/test_cache.rb index aecbdef3..aff0c426 100644 --- a/test/test_cache.rb +++ b/test/test_cache.rb @@ -1,7 +1,4 @@ require_relative 'test_case' - -GooTest.configure_goo - require_relative 'models' class TestCache < MiniTest::Unit::TestCase @@ -29,6 +26,19 @@ def self.after_suite GooTestData.delete_test_case_data end + def test_cache_invalidate + address = Address.all.first + Goo.use_cache = true + puts "save 1" + University.new(name: 'test', address: [address]).save + u2 = University.new(name: 'test', address: [address]) + puts "request 1" + refute u2.valid? + expected_error = { :name => { :duplicate => "There is already a persistent resource with id `http://goo.org/default/university/test`" } } + assert_equal expected_error, u2.errors + Goo.use_cache = false + end + def test_cache_models redis = Goo.redis_client redis.flushdb @@ -52,7 +62,7 @@ def test_cache_models assert !key.nil? assert redis.exists(key) - + prg = programs.first prg.bring_remaining prg.credits = 999 @@ -103,7 +113,7 @@ def test_cache_models_back_door data = " " + " " + " ." - + Goo.sparql_data_client.append_triples(Student.type_uri,data,"application/x-turtle") programs = Program.where(name: "BioInformatics", university: [ name: "Stanford" ]) .include(:students).all @@ -131,15 +141,20 @@ def x.response_backup *args def x.response *args raise Exception, "Should be a successful hit" end - programs = Program.where(name: "BioInformatics", university: [ name: "Stanford" ]) - .include(:students).all + begin + programs = Program.where(name: "BioInformatics", university: [ name: "Stanford" ]) + .include(:students).all + rescue Exception + assert false, "should be cached" + end + #from cache - assert programs.length == 1 - assert_raises Exception do + assert_equal 1, programs.length + assert_raises Exception do #different query programs = Program.where(name: "BioInformatics X", university: [ name: "Stanford" ]).all end - Goo.test_reset + TestHelpers.test_reset Goo.use_cache=false end diff --git a/test/test_case.rb b/test/test_case.rb index cf25de12..a8eda0ab 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -1,6 +1,13 @@ -# Start simplecov if this is a coverage task -if ENV["COVERAGE"].eql?("true") - require 'simplecov' +# Start simplecov if this is a coverage task or if it is run in the CI pipeline +if ENV["COVERAGE"] == "true" || ENV["CI"] == "true" + require "simplecov" + require "simplecov-cobertura" + # https://github.com/codecov/ruby-standard-2 + # Generate HTML and Cobertura reports which can be consumed by codecov uploader + SimpleCov.formatters = SimpleCov::Formatter::MultiFormatter.new([ + SimpleCov::Formatter::HTMLFormatter, + SimpleCov::Formatter::CoberturaFormatter + ]) SimpleCov.start do add_filter "/test/" add_filter "app.rb" @@ -13,16 +20,91 @@ MiniTest::Unit.autorun require_relative "../lib/goo.rb" +require_relative '../config/config.test' + +# Safety guard for destructive tests: ensure test targets are safe (localhost or -ut suffix) +module TestSafety + SAFE_HOSTS = Regexp.new(/localhost|-ut/) + MAX_REDIS_KEYS = 10 + + def self.safe_host?(value) + value = value.to_s + return false if value.empty? + !!(value =~ SAFE_HOSTS) + end + + def self.targets + { + triplestore: Goo.settings.goo_host.to_s, + search: Goo.settings.search_server_url.to_s, + redis: Goo.settings.goo_redis_host.to_s + } + end + + def self.unsafe_targets? + t = targets + unsafe = !safe_host?(t[:triplestore]) || !safe_host?(t[:search]) || !safe_host?(t[:redis]) + [unsafe, t] + end + + def self.ensure_safe_test_targets! + return if @safety_checked + unsafe, t = unsafe_targets? + return if !unsafe || ENV['CI'] == 'true' + + if $stdin.tty? + puts "\n\n================================== WARNING ==================================\n" + puts "** TESTS CAN BE DESTRUCTIVE -- YOU ARE POINTING TO A POTENTIAL PRODUCTION/STAGE SERVER **" + puts "Servers:" + puts "triplestore -- #{t[:triplestore]}" + puts "search -- #{t[:search]}" + puts "redis -- #{t[:redis]}" + print "Type 'y' to continue: " + $stdout.flush + confirm = $stdin.gets + abort('Canceling tests...') unless confirm && confirm.strip == 'y' + puts 'Running tests...' + $stdout.flush + else + abort('Aborting tests: non-whitelisted targets and non-interactive session.') + end + ensure + @safety_checked = true + end + + def self.ensure_safe_redis_size! + redis = Goo.redis_client + return unless redis + count = redis.dbsize + return if count <= MAX_REDIS_KEYS + abort("Aborting tests: redis has #{count} keys, expected <= #{MAX_REDIS_KEYS} for a test instance.") + end +end + +TestSafety.ensure_safe_test_targets! + +module TestHelpers + def self.test_reset + TestSafety.ensure_safe_test_targets! + TestSafety.ensure_safe_redis_size! + Goo.class_variable_set(:@@sparql_backends, {}) + Goo.add_sparql_backend(:main, + backend_name: Goo.settings.goo_backend_name, + query: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_query}", + data: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_data}", + update: "http://#{Goo.settings.goo_host}:#{Goo.settings.goo_port}#{Goo.settings.goo_path_update}", + options: { rules: :NONE }) + end +end class GooTest class Unit < MiniTest::Unit + def before_suites - # code to run before the first test (gets inherited in sub-tests) end def after_suites - # code to run after the last test (gets inherited in sub-tests) end def _run_suites(suites, type) @@ -35,48 +117,23 @@ def _run_suites(suites, type) end def _run_suite(suite, type) - [1,5,10,200].each do |slice_size| + ret = [] + [Goo.slice_loading_size].each do |slice_size| puts "\nrunning test with slice_loading_size=#{slice_size}" Goo.slice_loading_size=slice_size begin suite.before_suite if suite.respond_to?(:before_suite) - super(suite, type) + ret += super(suite, type) ensure suite.after_suite if suite.respond_to?(:after_suite) end end + return ret end end MiniTest::Unit.runner = GooTest::Unit.new - def self.configure_goo - if not Goo.configure? - Goo.configure do |conf| - conf.add_redis_backend(:host => "localhost") - conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/")) - conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#")) - conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#")) - conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#")) - conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"),default=true) - conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/")) - conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/")) - conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#")) - conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#")) - conf.add_namespace(:bioportal, RDF::Vocabulary.new("http://data.bioontology.org/")) - conf.add_namespace(:nemo, RDF::Vocabulary.new( - "http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#")) - - conf.add_sparql_backend(:main, query: "http://localhost:8080/sparql/", - data: "http://localhost:8080/data/", - update: "http://localhost:8080/update/", - options: { rules: :NONE }) - conf.add_search_backend(:main, service: "http://localhost:8983/solr/term_search_core1" ) - conf.use_cache=false - end - end - end - def self.triples_for_subject(resource_id) rs = Goo.sparql_query_client.query("SELECT * WHERE { #{resource_id.to_ntriples} ?p ?o . }") count = 0 @@ -97,4 +154,3 @@ def self.count_pattern(pattern) end end - diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb index 388f778a..9b2294e2 100644 --- a/test/test_chunks_write.rb +++ b/test/test_chunks_write.rb @@ -1,11 +1,8 @@ require_relative 'test_case' -GooTest.configure_goo - module TestChunkWrite - - ONT_ID = "http:://example.org/data/nemo" - ONT_ID_EXTRA = "http:://example.org/data/nemo/extra" + ONT_ID = "http://example.org/data/nemo" + ONT_ID_EXTRA = "http://example.org/data/nemo/extra" class TestChunkWrite < MiniTest::Unit::TestCase @@ -13,7 +10,6 @@ def initialize(*args) super(*args) end - def self.before_suite _delete end @@ -22,84 +18,67 @@ def self.after_suite _delete end + def setup + self.class._delete + end + + def self._delete - graphs = [ONT_ID,ONT_ID_EXTRA] - url = Goo.sparql_data_client.url - graphs.each do |graph| - #this bypasses the chunks stuff - params = { - method: :delete, - url: "#{url.to_s}#{graph.to_s}", - timeout: nil - } - RestClient::Request.execute(params) - end + graphs = [ONT_ID, ONT_ID_EXTRA] + graphs.each { |graph| Goo.sparql_data_client.delete_graph(graph) } end def test_put_data graph = ONT_ID ntriples_file_path = "./test/data/nemo_ontology.ntriples" + triples_no_bnodes = 25256 - result = Goo.sparql_data_client.put_triples( - graph, - ntriples_file_path, - mime_type="application/x-turtle") + Goo.sparql_data_client.put_triples(graph, ntriples_file_path, mime_type="application/x-turtle") - triples_no_bnodes = 25293 count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s ?p ?o }}" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == triples_no_bnodes + assert_equal triples_no_bnodes, sol[:c].object end - count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s ?p ?o ." - count += " FILTER(isBlank(?s)) }}" + + count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s ?p ?o . FILTER(isBlank(?s)) }}" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == 0 + assert_equal 0, sol[:c].object end end def test_put_delete_data graph = ONT_ID ntriples_file_path = "./test/data/nemo_ontology.ntriples" + triples_no_bnodes = 25256 - result = Goo.sparql_data_client.put_triples( - graph, - ntriples_file_path, - mime_type="application/x-turtle") + Goo.sparql_data_client.put_triples(graph, ntriples_file_path, mime_type="application/x-turtle") - triples_no_bnodes = 25293 count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s ?p ?o }}" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == triples_no_bnodes + assert_equal triples_no_bnodes, sol[:c].object end - puts "starting to delete" - result = Goo.sparql_data_client.delete_graph(graph) + + puts "Starting deletion" + Goo.sparql_data_client.delete_graph(graph) + puts "Deletion complete" + count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s ?p ?o }}" - puts "deleted completed" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == 0 + assert_equal 0, sol[:c].object end end def test_reentrant_queries ntriples_file_path = "./test/data/nemo_ontology.ntriples" - #by pass in chunks - url = Goo.sparql_data_client.url - params = { - method: :put, - url: "#{url.to_s}#{ONT_ID}", - payload: File.read(ntriples_file_path), - headers: {content_type: "application/x-turtle"}, - timeout: nil - } + + # Bypass in chunks + params = self.class.params_for_backend(:post, ONT_ID, ntriples_file_path) RestClient::Request.execute(params) tput = Thread.new { - result = Goo.sparql_data_client.put_triples( - ONT_ID_EXTRA, - ntriples_file_path, - mime_type="application/x-turtle") + Goo.sparql_data_client.put_triples(ONT_ID_EXTRA, ntriples_file_path, mime_type="application/x-turtle") } - sleep(1.5) + count_queries = 0 tq = Thread.new { 5.times do @@ -110,22 +89,21 @@ def test_reentrant_queries count_queries += 1 end } - tq.join assert tput.alive? - assert count_queries == 5 + assert_equal 5, count_queries tput.join - triples_no_bnodes = 25293 + count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID_EXTRA}> { ?s ?p ?o }}" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == triples_no_bnodes + assert_includes [25256, 50512], sol[:c].object end tdelete = Thread.new { Goo.sparql_data_client.delete_graph(ONT_ID_EXTRA) } - sleep(1.5) + count_queries = 0 tq = Thread.new { 5.times do @@ -137,58 +115,63 @@ def test_reentrant_queries end } tq.join - assert tdelete.alive? - assert count_queries == 5 tdelete.join + assert_equal 5, count_queries + count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID_EXTRA}> { ?s ?p ?o }}" Goo.sparql_query_client.query(count).each do |sol| - assert sol[:c].object == 0 + assert_equal 0, sol[:c].object end end def test_query_flood ntriples_file_path = "./test/data/nemo_ontology.ntriples" - #by pass in chunks - url = Goo.sparql_data_client.url - params = { - method: :put, - url: "#{url.to_s}#{ONT_ID}", - payload: File.read(ntriples_file_path), - headers: {content_type: "application/x-turtle"}, - timeout: nil - } + params = self.class.params_for_backend(:post, ONT_ID, ntriples_file_path) RestClient::Request.execute(params) tput = Thread.new { - result = Goo.sparql_data_client.put_triples( - ONT_ID_EXTRA, - ntriples_file_path, - mime_type="application/x-turtle") + Goo.sparql_data_client.put_triples(ONT_ID_EXTRA, ntriples_file_path, mime_type="application/x-turtle") } + threads = [] 25.times do |i| threads << Thread.new { 50.times do |j| - oq = "SELECT (count(?s) as ?c) WHERE { ?s a ?o }" + # The query WHERE { ?s a ?o } does not specify a graph, so it runs against the default graph. + # In AllegroGraph, the default graph is empty by default and does not include named graphs. + # In 4store/Virtuoso, the default graph is effectively a union of named graphs, + # so the original query works. Therefore, in AllegroGraph the count returns 0, causing + # refute_equal 0 to fail. This commit adds a named graph to the query. + oq = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID}> { ?s a ?o } }" Goo.sparql_query_client.query(oq).each do |sol| - assert sol[:c].object > 0 + refute_equal 0, sol[:c].to_i end end } end - log_status = [] - Thread.new { - 10.times do |i| - log_status << Goo.sparql_query_client.status - sleep(1.2) + + threads.each(&:join) + + if Goo.backend_4s? + log_status = [] + status_thread = Thread.new { + 10.times do |i| + log_status << Goo.sparql_query_client.status + end + } + + threads.each do |t| + t.join end - } - threads.each do |t| - t.join + tput.join + status_thread.join + + assert_equal 16, log_status.map { |x| x[:running] }.max end - tput.join - assert log_status.map { |x| x[:outstanding] }.max > 0 - assert log_status.map { |x| x[:running] }.max == 16 + end + + def self.params_for_backend(method, graph_name, ntriples_file_path = nil) + Goo.sparql_data_client.params_for_backend(graph_name, File.read(ntriples_file_path), "text/turtle", method) end end diff --git a/test/test_collections.rb b/test/test_collections.rb index 390ad349..65d1f46d 100644 --- a/test/test_collections.rb +++ b/test/test_collections.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - #collection on attribute class Issue < Goo::Base::Resource model :issue, collection: :owner, name_with: :description @@ -91,18 +89,18 @@ def test_unique_per_collection def test_inverse_on_collection skip "Not supported inverse on collection" - john = User.find("John").include(:name).first || - User.new(name: "John").save() + john = User.find("John").include(:name).first || User.new(name: "John").save + 5.times do |i| - Issue.new(description: "issue_#{i}", owner: john).save + Issue.find("issue_#{i}").in(john) || Issue.new(description: "issue_#{i}", owner: john).save end - - binding.pry - User.find("John",include: [:issues]).first.issues - User.find("John",include: [issues: [:desciption]]).first.issues - 5.times do |i| - Issue.find("issue_#{i}", collection: john).delete + issues = User.find("John").include(:issues).first.issues + assert_equal 5, issues.size + + issues.each do |issue| + assert_equal "issue_#{i}", issue.description + assert_equal john, issue.collection end end diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb index daba73e6..05ad1133 100644 --- a/test/test_dsl_settings.rb +++ b/test/test_dsl_settings.rb @@ -1,6 +1,21 @@ require_relative 'test_case' -GooTest.configure_goo + +class NewPersonModel < Goo::Base::Resource + model :person_model_new, name_with: :name + attribute :name, type: :string, enforce: [ :existence, :unique] + attribute :multiple_values, type: [:list, :integer], enforce: [ :existence, :min_3, :max_5 ] + attribute :one_number, type: :integer,enforce: [ :existence ] #by default not a list + attribute :birth_date, type: :date_time, enforce: [ :existence ] + + attribute :created, type: DateTime , + default: lambda { |record| DateTime.now }, + namespace: :omv + + attribute :friends, type: NewPersonModel , enforce: [ :existence] + attribute :status, type: :status, enforce: [ :existence], + default: lambda { |record| StatusModel.find("single") } +end class StatusModel < Goo::Base::Resource model :status_model, name_with: :name @@ -32,13 +47,83 @@ def initialize(attributes = {}) end end -class TestDSLSeeting < MiniTest::Unit::TestCase + +class YamlSchemeModelTest < Goo::Base::Resource + model :yaml_scheme_model_test, name_with: :name, scheme: 'test/data/yaml_scheme_model_test.yml' + attribute :name, enforce: [ :existence, :string, :unique] + attribute :last_name, enforce: [ :existence, :string, :unique] + attribute :birth_date, enforce: [ :existence, :date_time ] + attribute :nationality, enforce: [ :existence, :string ] + attribute :created, enforce: [ DateTime ], + default: lambda { |record| DateTime.now }, + namespace: :omv + attribute :friends, enforce: [ :existence , PersonModel] + attribute :status, enforce: [ :existence, :status ], + default: lambda { |record| StatusModel.find("single") } + + attribute :test_string, enforce: [:string] + attribute :test_integer, enforce: [:integer] + attribute :test_list, enforce: [:list] + attribute :test_float, enforce: [:float] + attribute :test_uri, enforce: [:uri] + attribute :test_boolean, enforce: [:boolean] +end + + +class TestDSLSetting < MiniTest::Unit::TestCase def initialize(*args) super(*args) end + def test_data_type_dsl + _test_attributes_enforce NewPersonModel + end + def test_attributes_set_get + _test_attributes_enforce PersonModel + end + + def test_default_value + #default is on save ... returns` person = PersonModel.new + assert_equal nil, person.created + end + + def test_model_with_yaml_scheme + + settings = YamlSchemeModelTest.model_settings + attributes_settings = settings[:attributes] + + + assert_equal "test/data/yaml_scheme_model_test.yml", settings[:scheme] + + assert_equal 'Name', attributes_settings[:name][:label] + assert_equal 'Person name', attributes_settings[:name][:description] + assert_equal %w[test:name test2:name test3:person_name], attributes_settings[:name][:equivalents] + assert_equal 'Put the person name as string', attributes_settings[:name][:help] + assert_equal 'John', attributes_settings[:name][:example] + + + assert_equal 'Person nationality', attributes_settings[:nationality][:label] + hash = {fr: 'france', us: 'USA'} + assert_equal hash, attributes_settings[:nationality][:enforcedValues] + end + + def test_default_value_with_yaml_scheme + settings = YamlSchemeModelTest.model_settings + attributes_settings = settings[:attributes] + assert_equal 'Test String', attributes_settings[:test_string][:default] + assert_equal 2, attributes_settings[:test_integer][:default] + assert_equal ["item1", "item2"], attributes_settings[:test_list][:default] + assert_equal 3.14, attributes_settings[:test_float][:default] + assert_equal "https://example.com/term1", attributes_settings[:test_uri][:default] + assert_equal false, attributes_settings[:test_boolean][:default] + end + + private + def _test_attributes_enforce(model) + person = model.new + model_key_name = model.model_name assert(person.respond_to? :id) assert(person.kind_of? Goo::Base::Resource) assert !person.valid? @@ -67,7 +152,7 @@ def test_attributes_set_get assert !person.valid? assert !person.errors[:birth_date] - person.birth_date = "X" + person.birth_date = "X" assert !person.valid? assert person.errors[:birth_date][:date_time] @@ -99,21 +184,21 @@ def test_attributes_set_get assert !person.valid? assert !person.errors[:multiple_values] - assert_raises RuntimeError do - person.multiple_values << 99 #RuntimeError: can't modify frozen Array + assert_raises FrozenError do + person.multiple_values << 99 end - friends = [PersonModel.new , PersonModel.new] + friends = [model.new , model.new] person.friends = friends assert !person.valid? assert person.errors[:friends][:no_list] - person.friends = PersonModel.new + person.friends = model.new assert !person.valid? - assert person.errors[:friends][:person_model] + assert person.errors[:friends][model_key_name] person.friends = "some one" assert !person.valid? - assert person.errors[:friends][:person_model] - person.friends = PersonModel.new + assert person.errors[:friends][model_key_name] + person.friends = model.new person.one_number = 99 assert !person.valid? @@ -127,7 +212,7 @@ def test_attributes_set_get assert !person.valid? assert person.errors[:one_number][:no_list] - person.one_number = 99 + person.one_number = 99 assert_equal(99, person.one_number) assert !person.valid? assert !person.errors[:one_number] @@ -139,10 +224,5 @@ def test_attributes_set_get assert !person.valid? end - def test_default_value - #default is on save ... returns` - person = PersonModel.new - assert_equal nil, person.created - end end diff --git a/test/test_email_validator.rb b/test/test_email_validator.rb new file mode 100644 index 00000000..eed1572b --- /dev/null +++ b/test/test_email_validator.rb @@ -0,0 +1,85 @@ +require_relative 'test_case.rb' + +module Goo + module Validators + class TestEmail < MiniTest::Unit::TestCase + + def dummy_instance + @dummy ||= Object.new + end + + def validate(value) + Email.new(dummy_instance, :email, value) + end + + def assert_valid(value) + validator = validate(value) + assert validator.valid?, "Expected #{value.inspect} to be valid" + end + + def assert_invalid(value) + validator = validate(value) + refute validator.valid?, "Expected #{value.inspect} to be invalid" + end + + def test_valid_emails + assert_valid nil + assert_valid "user@example.com" + assert_valid "john.doe+test@sub.domain.org" + assert_valid "a_b-c@foo-bar.co.uk" + assert_valid "user123@domain.io" + end + + def test_invalid_emails_structure + assert_invalid "" + assert_invalid "plainaddress" + assert_invalid "user@localhost" + assert_invalid "user@com" + assert_invalid "user@.com" + assert_invalid "user@com." + assert_invalid "user@-domain.com" + assert_invalid "user@domain-.com" + assert_invalid "user.@example.com" + assert_invalid "user..user@example.com" + assert_invalid "user@domain..com" + assert_invalid "user@" + end + + def test_email_length_limits + too_short = "a@b.c" # 5 chars + assert_invalid too_short + + long_local = "a" * 65 + assert_invalid "#{long_local}@example.com" + + long_domain = ("a" * 63 + ".") * 4 + "com" + assert_invalid "user@#{long_domain}" + + too_long = "#{'a'*64}@#{'b'*189}.com" # 258 chars + assert_invalid too_long + end + + def test_array_with_all_valid_emails + validator = validate(["valid@example.com", "foo.bar@domain.co"]) + assert validator.valid? + end + + def test_array_with_one_invalid_email + validator = validate(["good@domain.com", "bad@domain..com"]) + refute validator.valid? + end + + def test_error_message_for_single_invalid_email + validator = validate("invalid-email") + refute validator.valid? + assert_match(/must be a valid email address/i, validator.error) + end + + def test_error_message_for_array_with_invalid + validator = validate(["invalid@", "also@bad"]) + refute validator.valid? + assert_match(/All values.*must be valid email addresses/i, validator.error) + end + end + end +end \ No newline at end of file diff --git a/test/test_enum.rb b/test/test_enum.rb index db41c343..eaf13af2 100644 --- a/test/test_enum.rb +++ b/test/test_enum.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestEnum VALUES = ["uploaded","removed","archived"] diff --git a/test/test_index.rb b/test/test_index.rb index 4d781973..bf4b8937 100644 --- a/test/test_index.rb +++ b/test/test_index.rb @@ -1,8 +1,6 @@ require_relative 'test_case' require_relative './app/models' -GooTest.configure_goo - module TestIndex class TestSchemaless < MiniTest::Unit::TestCase diff --git a/test/test_inmutable.rb b/test/test_inmutable.rb deleted file mode 100644 index 9d6037c0..00000000 --- a/test/test_inmutable.rb +++ /dev/null @@ -1,102 +0,0 @@ -require_relative 'test_case' - -GooTest.configure_goo - -module TestInmutable - class Status < Goo::Base::Resource - model :status, :inmutable, name_with: :code - attribute :code, enforce: [:unique, :existence] - attribute :description, enforce: [:existence] - end - - class Person < Goo::Base::Resource - model :person, :inmutable, name_with: :name - attribute :name, enforce: [:unique, :existence] - attribute :status, enforce: [:status, :existence] - end - - class TestInmutableCase < MiniTest::Unit::TestCase - def initialize(*args) - super(*args) - end - - def setup - end - - def self.before_suite - status = ["single", "married", "divorced", "widowed"] - status.each do |st| - stt = Status.new(code: st, description: (st + " some desc")) - stt.save - end - people = [ - ["Susan","married"], - ["Lee","divorced"], - ["John","divorced"], - ["Peter","married"], - ["Christine","married"], - ["Ana","single"], - ] - people.each do |p| - po = Person.new - po.name = p[0] - po.status = Status.find(p[1]).first - po.save - end - end - - def self.after_suite - objs = [Person,Status] - objs.each do |obj| - obj.where.all.each do |st| - st.delete - end - end - end - - ## TODO inmutable are deprecated - they might come back in a different way" - def skip_test_inmutable - #they come fully loaded - Status.load_inmutable_instances - status1 = Status.where.all.sort_by { |s| s.code } - status2 = Status.where.all.sort_by { |s| s.code } - assert status1.length == 4 - assert status2.length == 4 - #same referencs - status1.each_index do |i| - assert status1[i].object_id==status2[i].object_id - end - - #create a new object - stt = Status.new(code: "xx", description: ("xx" + " some desc")) - stt.save - - status1 = Status.where.all.sort_by { |s| s.code } - status2 = Status.where.all.sort_by { |s| s.code } - assert status1.length == 5 - assert status2.length == 5 - #same referencs - status1.each_index do |i| - assert status1[i].object_id==status2[i].object_id - end - - status1.each do |st| - assert st.code - assert st.description - end - - marr = Status.find("divorced").first - assert marr.code == "divorced" - assert marr.description - assert marr.object_id == status1.first.object_id - - people = Person.where.include(:name, status: [ :code, :description ]).all - people.each do |p| - assert p.status.object_id == status1.select { |st| st.id == p.status.id }.first.object_id - assert p.status.code - assert p.status.description - end - end - - end -end diff --git a/test/test_inverse.rb b/test/test_inverse.rb index e926a572..2fbb4479 100644 --- a/test/test_inverse.rb +++ b/test/test_inverse.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class Task < Goo::Base::Resource model :task, name_with: :description attribute :description, enforce: [ :existence, :unique] diff --git a/test/test_languages_filters.rb b/test/test_languages_filters.rb new file mode 100644 index 00000000..9fa0ccff --- /dev/null +++ b/test/test_languages_filters.rb @@ -0,0 +1,91 @@ +require_relative "test_case" +require_relative './app/models' + + + +class ExamplePerson < Goo::Base::Resource + model :person, namespace: :bioportal, name_with: lambda { |k| k.id }, + collection: :db + attribute :db, enforce: [ :database ] + attribute :label, namespace: :rdf, enforce: [ :list ] +end + +class ExamplePlace < Goo::Base::Resource + model :place, namespace: :bioportal, name_with: lambda { |k| k.id }, + collection: :db + attribute :db, enforce: [ :database ] + attribute :label, namespace: :rdf, enforce: [ :list ] +end + +class TestLanguageFilter < MiniTest::Unit::TestCase + def self.before_suite + RequestStore.store[:requested_lang] = Goo.main_languages.first + graph = RDF::URI.new(Test::Models::DATA_ID) + + database = Test::Models::Database.new + database.id = graph + database.name = "Census tiger 2002" + database.save + + @@db = Test::Models::Database.find(RDF::URI.new(Test::Models::DATA_ID)).first + @@person_id = RDF::URI.new "http://data.bioontology.org/resource1" + + + ntriples_file_path = "./test/data/languages.nt" + + Goo.sparql_data_client.put_triples( + graph, + ntriples_file_path, + mime_type = "application/x-turtle") + end + + def self.after_suite + graph = RDF::URI.new(Test::Models::DATA_ID) + Goo.sparql_data_client.delete_graph(graph) + database = Test::Models::Database.find(RDF::URI.new(Test::Models::DATA_ID)).first + database.delete if database + RequestStore.store[:requested_lang] = Goo.main_languages.first + end + + def setup + RequestStore.store[:requested_lang] = Goo.main_languages.first + end + + def test_one_language + # by default english and not tagged values + person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first + assert_equal ["John Doe", "Juan Pérez"].sort, person.label.sort + + + # select french, return french values and not tagged values + RequestStore.store[:requested_lang] = :fr + person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first + assert_equal ["Jean Dupont", "Juan Pérez"].sort, person.label.sort + + end + + def test_multiple_languages + # select all languages + RequestStore.store[:requested_lang] = :all + expected_result = {:en=>["John Doe"], :fr=>["Jean Dupont"], "@none"=>["Juan Pérez"]} + person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first + assert_equal expected_result.values.flatten.sort, person.label.sort + + # using include_languages on any attribute returns an hash of {language: values} instead of the array of values + assert_equal expected_result, person.label(include_languages: true) + + # filter only french, english and not tagged values + RequestStore.store[:requested_lang] = [:fr, :en] + person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first + assert_equal expected_result.values.flatten.sort.sort, person.label.sort + assert_equal expected_result, person.label(include_languages: true) + end + + + def test_language_not_found + RequestStore.store[:requested_lang] = :ar + person = ExamplePerson.find(@@person_id).in(@@db).include(:label).first + # will return only not tagged values if existent + assert_equal ["Juan Pérez"], person.label + end +end diff --git a/test/test_logging.rb b/test/test_logging.rb new file mode 100644 index 00000000..92018357 --- /dev/null +++ b/test/test_logging.rb @@ -0,0 +1,54 @@ +require_relative 'test_case' +require_relative 'models' + +class TestLogging < MiniTest::Unit::TestCase + + def self.before_suite + GooTestData.create_test_case_data + Goo.use_cache = true + Goo.redis_client.flushdb + Goo.add_query_logger(enabled: true, file: "test.log") + end + + def self.after_suite + GooTestData.delete_test_case_data + Goo.add_query_logger(enabled: false, file: nil) + File.delete("test.log") if File.exist?("test.log") + Goo.redis_client.flushdb + Goo.use_cache = false + end + + def setup + Goo.redis_client.flushdb + end + + def test_logging + Goo.logger.info("Test logging") + University.all + recent_logs = Goo.logger.get_logs + assert_equal 2, recent_logs.length + assert recent_logs.any? { |x| x['query'].include?("Test logging") } + assert File.read("test.log").include?("Test logging") + end + + def test_last_10s_logs + Goo.logger.info("Test logging 2") + University.all + recent_logs = Goo.logger.queries_last_n_seconds(1) + assert_equal 2, recent_logs.length + assert recent_logs.any? { |x| x['query'].include?("Test logging 2") } + assert File.read("test.log").include?("Test logging 2") + sleep 1 + recent_logs = Goo.logger.queries_last_n_seconds(1) + assert_equal 0, recent_logs.length + end + + def test_auto_clean_logs + Goo.logger.info("Test logging 3") + (1..3000).each do |_i| + University.all + end + recent_logs = Goo.logger.get_logs + assert recent_logs.length < 2000 + end +end diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb index 4020ac3a..b11a348c 100644 --- a/test/test_model_complex.rb +++ b/test/test_model_complex.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestComplex class Submission < Goo::Base::Resource @@ -10,7 +8,7 @@ class Submission < Goo::Base::Resource end class Term < Goo::Base::Resource - model :class, + model :term, namespace: :owl, collection: :submission, name_with: :id, @@ -25,22 +23,22 @@ class Term < Goo::Base::Resource attribute :parents, namespace: :rdfs, property: lambda { |x| tree_property(x) }, - enforce: [:list, :class] + enforce: [:list, :term] attribute :ancestors, namespace: :rdfs, property: lambda { |x| tree_property(x) }, - enforce: [:list, :class], transitive: true + enforce: [:list, :term], transitive: true attribute :children, namespace: :rdfs, property: lambda { |x| tree_property(x) }, - inverse: { on: :class , attribute: :parents } + inverse: { on: :term , attribute: :parents } attribute :descendants, namespace: :rdfs, property: lambda { |x| tree_property(x) }, - inverse: { on: :class , attribute: :parents }, + inverse: { on: :term , attribute: :parents }, transitive: true def self.tree_property(*args) @@ -48,7 +46,7 @@ def self.tree_property(*args) if collection.id.to_s["submission1"] return RDF::RDFS[:subClassOf] end - return RDF::SKOS[:broader] + return RDF::Vocab::SKOS[:broader] end def self.class_rdf_type(*args) @@ -56,7 +54,7 @@ def self.class_rdf_type(*args) if collection.id.to_s["submission1"] return RDF::OWL[:Class] end - return RDF::SKOS[:Concept] + return RDF::Vocab::SKOS[:Concept] end attribute :methodBased, namespace: :rdfs, property: :subClassOf, handler: :dataMethod @@ -78,12 +76,13 @@ def self.before_suite if GooTest.count_pattern("?s ?p ?o") > 100000 raise Exception, "Too many triples in KB, does not seem right to run tests" end - Goo.sparql_update_client.update("DELETE {?s ?p ?o } WHERE { ?s ?p ?o }") + + Goo.sparql_data_client.delete_graph(Submission.uri_type.to_s) end def self.after_suite Goo.use_cache = false - Goo.sparql_update_client.update("DELETE {?s ?p ?o } WHERE { ?s ?p ?o }") + Goo.sparql_data_client.delete_graph(Submission.uri_type.to_s) end def test_method_handler @@ -99,19 +98,26 @@ def test_method_handler x.id = RDF::URI.new "http://someiri.org/term/x" x.prefLabel = "x" x.save - assert_raises ArgumentError do - y = Term.find(x.id).in(sub).include(:methodBased).first - end + # Chech the methodBased is not included + y = Term.find(x.id).in(sub).include(:methodBased).first + assert_kind_of TestComplex::Term, y + refute y.loaded_attributes.include?(:methodBased) + assert_raises ArgumentError do y = Term.find(x.id).in(sub).include(methodBased: [:prefLabel]).first end - assert_raises ArgumentError do - y = Term.where.in(sub).include(:methodBased).all - end + + # Chech there is result and the methodBased is not included + y = Term.where.in(sub).include(:methodBased).all + assert_kind_of Array, y + refute_empty y + assert_kind_of TestComplex::Term, y.first + refute y.first.loaded_attributes.include?(:methodBased) + + # Chech the methodBased is brought by the bring y = Term.find(x.id).in(sub).first - assert_raises ArgumentError do - y.bring(:methodBased) - end + y.bring(:methodBased) + assert_includes y.loaded_attributes.to_a, :methodBased y.delete sub.delete end @@ -185,6 +191,11 @@ def test_multiple_collection() def test_collection() + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -313,6 +324,11 @@ def test_two_resources_same_id def test_parents_inverse_children + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -320,6 +336,7 @@ def test_parents_inverse_children submission = Submission.find("submission1").first end + terms = Term.in(submission) terms.each do |t| t.delete @@ -342,7 +359,7 @@ def test_parents_inverse_children van.synonym = ["cargo", "syn van"] van.definition = ["vehicle def 1", "vehicle def 2"] van.parents = [vehicle] - assert van.valid? + assert van.valid?, "Invalid term: [id: #{van.id}, errors: #{van.errors}]" van.save assert_equal 1, GooTest.count_pattern( @@ -482,18 +499,20 @@ def test_empty_attributes end def test_aggregate + skip "Transitive closure doesn't work yet. AllegroGraph?" submission = Submission.new(name: "submission1") unless submission.exist? submission.save else submission = Submission.find("submission1").first end + terms = Term.in(submission) terms.each do |t| t.delete - assert_equal(0, - GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }")) + assert_equal 0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }") end + terms = [] 10.times do |i| term = Term.new @@ -510,127 +529,109 @@ def test_aggregate elsif i > 0 term.parents = [terms[5]] end - assert term.valid? + assert term.valid?, "Invalid term: [id: #{term.id}, errors: #{term.errors}]" term.save terms << term end terms = Term.in(submission).aggregate(:count, :children).all terms = terms.sort_by { |x| x.id } - assert terms[0].aggregates.first.value == 4 - assert terms[1].aggregates.first.value == 1 - assert terms[2].aggregates.first.value == 3 - assert terms[3].aggregates.first.value == 3 - assert terms[-1].aggregates.first.value == 0 - - page = Term.in(submission).include(:synonym, :prefLabel) - .aggregate(:count, :children) - .page(1) + assert_equal 4, terms[0].aggregates.first.value + assert_equal 1, terms[1].aggregates.first.value + assert_equal 3, terms[2].aggregates.first.value + assert_equal 3, terms[3].aggregates.first.value + assert_equal 0, terms[-1].aggregates.first.value + + page = Term.in(submission).include(:synonym, :prefLabel).aggregate(:count, :children).page(1) page.each do |t| if t.id.to_s.include? "term/0" - assert t.aggregates.first.value == 4 + assert_equal 4, t.aggregates.first.value elsif t.id.to_s.include? "term/1" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/2" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/3" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/9" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value end end - #with a parent - page = Term.where(parents: terms[0]).in(submission).include(:synonym, :prefLabel) - .aggregate(:count, :children) - .page(1) - assert page.length == 4 + # With a parent + page = Term.where(parents: terms[0]).in(submission) + .include(:synonym, :prefLabel).aggregate(:count, :children).page(1) + assert_equal 4, page.length page.each do |t| if t.id.to_s.include? "term/1" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/2" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/3" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/4" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value else - assert 1==0 + assert 1 == 0 end end - #with parent and query options - page = Term.where(ancestors: terms[0]) - .in(submission) - .include(:synonym, :prefLabel) - .aggregate(:count, :children) - .page(1) - - assert page.count == 9 + # With parent and query options + page = Term.where(ancestors: terms[0]).in(submission) + .include(:synonym, :prefLabel).aggregate(:count, :children).page(1) + assert_equal 9, page.count page.each do |t| if t.id.to_s.include? "term/1" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/2" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/3" - assert t.aggregates.first.value == 3 + assert_equal 3, t.aggregates.first.value elsif t.id.to_s.include? "term/4" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value elsif t.id.to_s.include? "term/5" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/6" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value elsif t.id.to_s.include? "term/7" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value elsif t.id.to_s.include? "term/8" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value elsif t.id.to_s.include? "term/9" - assert t.aggregates.first.value == 0 + assert_equal 0, t.aggregates.first.value else - assert 1==0 + assert 1 == 0 end end - #the other direction UP and query options and read only - page = Term.where(descendants: terms[9]) - .in(submission) - .include(:synonym, :prefLabel) - .aggregate(:count, :children) - .page(1) - - assert page.count == 3 + # The other direction UP, and query options, and read only + page = Term.where(descendants: terms[9]).in(submission) + .include(:synonym, :prefLabel).aggregate(:count, :children).page(1) + assert_equal 3, page.count page.each do |t| if t.id.to_s.include? "term/0" - assert t.aggregates.first.value == 4 + assert_equal 4, t.aggregates.first.value elsif t.id.to_s.include? "term/5" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/1" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value else - assert 1==0 + assert 1 == 0 end end - #with read only - ts = Term.where(descendants: terms[9]) - .in(submission) - .include(:synonym, :prefLabel) - .read_only - assert ts.length == 3 + # With read only + ts = Term.where(descendants: terms[9]).in(submission).include(:synonym, :prefLabel).read_only + assert_equal 3, ts.length ts.each do |t| assert_instance_of String, t.prefLabel - assert t.klass == Term - assert t.id.class == RDF::URI + assert_equal Term, t.klass + assert_equal RDF::URI, t.id.class assert_instance_of Array, t.synonym end - #read_only + page - ts = Term.where(descendants: terms[9]) - .in(submission) - .include(:synonym, :prefLabel) - .read_only - .page(1) - assert ts.length == 3 + # Read_only + page + ts = Term.where(descendants: terms[9]).in(submission).include(:synonym, :prefLabel).read_only.page(1) + assert_equal 3, ts.length ts.each do |t| assert_instance_of String, t.prefLabel assert t.klass == Term @@ -638,23 +639,18 @@ def test_aggregate assert_instance_of Array, t.synonym end - page = Term.where(descendants: terms[9]) - .in(submission) - .include(:synonym, :prefLabel) - .aggregate(:count, :children) - .read_only - .page(1) - - assert page.count == 3 + page = Term.where(descendants: terms[9]).in(submission) + .include(:synonym, :prefLabel).aggregate(:count, :children).read_only.page(1) + assert_equal 3, page.count page.each do |t| if t.id.to_s.include? "term/0" - assert t.aggregates.first.value == 4 + assert_equal 4, t.aggregates.first.value elsif t.id.to_s.include? "term/5" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value elsif t.id.to_s.include? "term/1" - assert t.aggregates.first.value == 1 + assert_equal 1, t.aggregates.first.value else - assert 1==0 + assert 1 == 0 end end end @@ -667,12 +663,18 @@ def test_empty_pages else submission = Submission.find("submission1").first end + terms = Term.in(submission) terms.each do |t| t.delete - assert_equal(0, - GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }")) + assert_equal 0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }") end + + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + terms = [] 10.times do |i| term = Term.new @@ -682,7 +684,7 @@ def test_empty_pages if i >= 1 && i < 5 term.parents = [terms[0]] end - assert term.valid? + assert term.valid?, "Invalid term: [id: #{term.id}, errors: #{term.errors}]" term.save terms << term end @@ -695,10 +697,16 @@ def test_empty_pages .include(Term.attributes) .page(1) .all - assert page_terms.length == 0 + assert_equal 0, page_terms.length end def test_readonly_pages_with_include + + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -711,6 +719,7 @@ def test_readonly_pages_with_include assert_equal(0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }")) end + terms = [] 10.times do |i| term = Term.new @@ -722,7 +731,7 @@ def test_readonly_pages_with_include elsif i >= 2 term.parents = [terms[1]] end - assert term.valid? + assert term.valid?, "Invalid term: [id: #{term.id}, errors: #{term.errors}]" term.save terms << term end diff --git a/test/test_name_with.rb b/test/test_name_with.rb index 7ba4df42..c2f226a4 100644 --- a/test/test_name_with.rb +++ b/test/test_name_with.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class NameWith < Goo::Base::Resource model :name_with, name_with: lambda { |s| id_generator(s) } attribute :name, enforce: [ :existence, :string, :unique ] diff --git a/test/test_namespaces.rb b/test/test_namespaces.rb index 3c2b89ea..6c4bddc0 100644 --- a/test/test_namespaces.rb +++ b/test/test_namespaces.rb @@ -1,8 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - - class NamespacesModel < Goo::Base::Resource model :namespaces, namespace: :rdfs, name_with: :name attribute :name, enforce: [ :existence, :string, :unique ], namespace: :skos @@ -14,31 +11,36 @@ def self.id_generator(inst) end end -class TestNamespaces < GooTest +class TestNamespaces < MiniTest::Unit::TestCase def initialize(*args) super(*args) end + def setup + john = NamespacesModel.find("John").first + john.delete unless john.nil? + end + def test_namespaces ns = NamespacesModel.new(name: "John", description: "description", location: "CA") - assert ns.class.uri_type.to_s["http://www.w3.org/2000/01/rdf-schema#"] != nil - assert ns.class.attribute_uri(:name).to_s["http://www.w3.org/2004/02/skos/core#"] != nil - assert ns.class.attribute_uri(:description).to_s["http://xmlns.com/foaf/0.1/"] != nil - assert ns.class.attribute_uri(:location).to_s["http://goo.org/default/"] != nil + refute_nil ns.class.uri_type.to_s["http://www.w3.org/2000/01/rdf-schema#"] + refute_nil ns.class.attribute_uri(:name).to_s["http://www.w3.org/2004/02/skos/core#"] + refute_nil ns.class.attribute_uri(:description).to_s["http://xmlns.com/foaf/0.1/"] + refute_nil ns.class.attribute_uri(:location).to_s["http://www.w3.org/2000/01/rdf-schema#"] assert ns.valid? ns.save - assert_equal(1, count_pattern(" #{ns.id.to_ntriples} a #{ns.class.uri_type.to_ntriples} .")) - assert_equal(1, count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:name).to_ntriples} ?x .")) - assert_equal(1, count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:description).to_ntriples} ?x .")) - assert_equal(1, count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:location).to_ntriples} ?x .")) + assert_equal 1, GooTest.count_pattern(" #{ns.id.to_ntriples} a #{ns.class.uri_type.to_ntriples} .") + assert_equal 1, GooTest.count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:name).to_ntriples} ?x .") + assert_equal 1, GooTest.count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:description).to_ntriples} ?x .") + assert_equal 1, GooTest.count_pattern(" #{ns.id.to_ntriples} #{ns.class.attribute_uri(:location).to_ntriples} ?x .") - from_backend = NamespacesModel.find(ns.id, include: NamespacesModel.attributes) + from_backend = NamespacesModel.find(ns.id, include: NamespacesModel.attributes).first NamespacesModel.attributes.each do |attr| assert_equal ns.send("#{attr}"), from_backend.send("#{attr}") end from_backend.delete - assert !from_backend.exist? - assert 0, triples_for_subject(from_backend.id) + refute from_backend.exist? + assert_equal 0, GooTest.triples_for_subject(from_backend.id) end end diff --git a/test/test_read_only.rb b/test/test_read_only.rb index 621d54c2..9855decf 100644 --- a/test/test_read_only.rb +++ b/test/test_read_only.rb @@ -1,8 +1,6 @@ require_relative 'test_case' require_relative 'test_where' -GooTest.configure_goo - module TestReadOnly class TestReadOnlyWithStruct < TestWhere @@ -39,12 +37,17 @@ def test_struct_find end def test_embed_struct - skip "not yet" + students = Student.where(enrolled: [university: [name: "Stanford"]]) .include(:name) - .include(enrolled: [:name, university: [ :address ]]) + .include(enrolled: [:name, university: [ :address, :name ]]) .read_only.all - binding.pry + + assert_equal 3, students.size + students.each do |st| + assert st.enrolled.any? {|e| e.is_a?(Struct) && e.university.name.eql?('Stanford')} + end + end end end diff --git a/test/test_schemaless.rb b/test/test_schemaless.rb index f95a17d5..42084eb8 100644 --- a/test/test_schemaless.rb +++ b/test/test_schemaless.rb @@ -1,8 +1,6 @@ require_relative 'test_case' -GooTest.configure_goo - -module TestSChemaless +module TestSchemaless ONT_ID = "http:://example.org/data/nemo" @@ -118,6 +116,9 @@ def test_find_include_schemaless where = Klass.find(cognition_term).in(ontology).include(:unmapped) k = where.first enter = 0 + + assert k.unmapped.keys.include?(Goo.vocabulary(:nemo)[:definition]) + k.unmapped.each do |p,vals| if p.to_s == Goo.vocabulary(:nemo)[:synonym].to_s enter += 1 @@ -185,7 +186,19 @@ def test_index_order_by end end + + def test_all_pages_loop + ontology = Ontology.find(RDF::URI.new(ONT_ID)).first + page = 1 + count = 0 + begin + paging = Klass.in(ontology).page(page,50).all + count += paging.size + page = paging.next_page if paging.next? + end while(paging.next?) + assert_equal count, Klass.in(ontology).count + end def test_page_reuse_predicates ontology = Ontology.find(RDF::URI.new(ONT_ID)).first paging = Klass.in(ontology).include(:unmapped).page(1,100) @@ -208,7 +221,7 @@ def test_page_reuse_predicates all_ids << k.id end total += page.length - paging.page(page.next_page) if page.next? + paging.page(page.next_page, 100) if page.next? assert page.aggregate == 1713 end while(page.next?) assert all_ids.length == all_ids.uniq.length diff --git a/test/test_search.rb b/test/test_search.rb index 8164f376..9ca74ca6 100644 --- a/test/test_search.rb +++ b/test/test_search.rb @@ -1,61 +1,155 @@ require_relative 'test_case' -GooTest.configure_goo - module TestSearch - class TermSearch < Goo::Base::Resource - model :term_search, name_with: :id + model :term_search, name_with: lambda { |resource| uuid_uri_generator(resource) } attribute :prefLabel, enforce: [:existence] - attribute :synonym #array of strings - attribute :definition #array of strings + attribute :synonym, enforce: [:list] # array of strings + attribute :definition # array of strings attribute :submissionAcronym, enforce: [:existence] attribute :submissionId, enforce: [:existence, :integer] - # dummy attributes to validate non-searchable fileds + # Dummy attributes to validate non-searchable files attribute :semanticType attribute :cui + enable_indexing(:term_search) do | schema_generator | + schema_generator.add_field(:prefLabel, 'text_general', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:synonym, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:definition, 'string', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:submissionAcronym, 'string', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:submissionId, 'pint', indexed: true, stored: true, multi_valued: false) + schema_generator.add_field(:cui, 'text_general', indexed: true, stored: true, multi_valued: true) + schema_generator.add_field(:semanticType, 'text_general', indexed: true, stored: true, multi_valued: true) + + # Copy fields for term search + schema_generator.add_copy_field('prefLabel', '_text_') + # for exact search + schema_generator.add_copy_field('prefLabel', 'prefLabelExact') + + # Matches whole terms in the suggest text + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggest') + + # Will match from the left of the field, e.g. if the document field + # is "A brown fox" and the query is "A bro", it will match, but not "brown" + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggestEdge') + + # Matches any word in the input field, with implicit right truncation. + # This means that the field "A brown fox" will be matched by query "bro". + # We use this to get partial matches, but these would be boosted lower than exact and left-anchored + schema_generator.add_copy_field('prefLabel', 'prefLabelSuggestNgram') + + schema_generator.add_copy_field('synonym', '_text_') + schema_generator.add_copy_field('synonym', 'synonymExact') + schema_generator.add_copy_field('synonym', 'synonymSuggest') + schema_generator.add_copy_field('synonym', 'synonymSuggestEdge') + schema_generator.add_copy_field('synonym', 'synonymSuggestNgram') + end + def index_id() "#{self.id.to_s}_#{self.submissionAcronym}_#{self.submissionId}" end - def index_doc() + def index_doc(to_set = nil) self.to_hash end end + class TermSearch2 < Goo::Base::Resource + model :term_search2, name_with: :prefLabel + attribute :prefLabel, enforce: [:existence], fuzzy_search: true + attribute :synonym, enforce: [:list] + attribute :definition + attribute :submissionAcronym, enforce: [:existence] + attribute :submissionId, enforce: [:existence, :integer] + attribute :private, enforce: [:boolean], default: false, index: false + # Dummy attributes to validate non-searchable files + attribute :semanticType + attribute :cui + + enable_indexing(:test_solr) + end + + class TermSearch3 < Goo::Base::Resource + model :term_search3, name_with: :prefLabel + attribute :prefLabel, enforce: [:existence] + attribute :synonym, enforce: [:list] + attribute :definition + attribute :submissionAcronym, enforce: [:existence] + attribute :submissionId, enforce: [:existence, :integer] + attribute :private, enforce: [:boolean], default: false, index: false + # Dummy attributes to validate non-searchable files + attribute :semanticType + attribute :cui + + attribute :object, enforce: [:term_search] + attribute :object_list, enforce: [:term_search, :list] + + + enable_indexing(:test_solr) + end + class TestModelSearch < MiniTest::Unit::TestCase + def self.before_suite + Goo.init_search_connections(true) + end def setup @terms = [ TermSearch.new( - :id => RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Melanoma"), - :prefLabel => "Melanoma", - :synonym => ["Cutaneous Melanoma", "Skin Cancer", "Malignant Melanoma"], - :definition => "Melanoma refers to a malignant skin cancer", - :submissionAcronym => "NCIT", - :submissionId => 2, - :semanticType => "Neoplastic Process", - :cui => "C0025202" + id: RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Melanoma"), + prefLabel: "Melanoma", + synonym: [ + "Cancerous Melanoma", + "Skin Cancer", + "Malignant Melanoma" + ], + definition: "Melanoma refers to a malignant skin cancer", + submissionAcronym: "NCIT", + submissionId: 2, + semanticType: "Neoplastic Process", + cui: "C0025202" + ), + TermSearch.new( + id: RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Neoplasm"), + prefLabel: "Neoplasm", + synonym: [ + "tumor", + "肿瘤", + "Neoplasms", + "NEOPLASMS BENIGN", + "MALIGNANT AND UNSPECIFIED (INCL CYSTS AND POLYPS)", + "Neoplasia", + "Neoplastic Growth" + ], + definition: "A benign or malignant tissue growth resulting from uncontrolled cell proliferation. "\ + "Benign neoplastic cells resemble normal cells without exhibiting significant cytologic atypia, while "\ + "malignant cells exhibit overt signs such as dysplastic features, atypical mitotic figures, necrosis, "\ + "nuclear pleomorphism, and anaplasia. Representative examples of benign neoplasms include papillomas, "\ + "cystadenomas, and lipomas; malignant neoplasms include carcinomas, sarcomas, lymphomas, and leukemias.", + submissionAcronym: "NCIT", + submissionId: 2, + semanticType: "Neoplastic Process", + cui: "C0375111" ), TermSearch.new( - :id => RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Neoplasm"), - :prefLabel => "Neoplasm", - :synonym => ["tumor", "Neoplasms", "NEOPLASMS BENIGN", "MALIGNANT AND UNSPECIFIED (INCL CYSTS AND POLYPS)", "Neoplasia", "Neoplastic Growth"], - :definition => "A benign or malignant tissue growth resulting from uncontrolled cell proliferation. Benign neoplastic cells resemble normal cells without exhibiting significant cytologic atypia, while malignant cells exhibit overt signs such as dysplastic features, atypical mitotic figures, necrosis, nuclear pleomorphism, and anaplasia. Representative examples of benign neoplasms include papillomas, cystadenomas, and lipomas; malignant neoplasms include carcinomas, sarcomas, lymphomas, and leukemias.", - :submissionAcronym => "NCIT", - :submissionId => 2, - :semanticType => "Neoplastic Process", - :cui => "C0375111" - ) + id: RDF::URI.new("http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#Melanoma2"), + prefLabel: "Melanoma with cutaneous melanoma syndrome", + synonym: [ + "Cutaneous Melanoma", + "Melanocytes Skin Cancer", + "Malignant Melanoma" + ], + definition: "Melanoma refers to a malignant skin cancer", + submissionAcronym: "NCIT", + submissionId: 2, + semanticType: "Neoplastic Process", + cui: "C0025202" + ), ] end - def teardown - end - def initialize(*args) super(*args) end @@ -69,6 +163,103 @@ def test_search assert_equal @terms[1].prefLabel, resp["response"]["docs"][0]["prefLabel"] end + def test_search_filters + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelExact^100 prefLabelSuggestEdge^50 synonymSuggestEdge^10 prefLabelSuggestNgram synonymSuggestNgram resource_id cui semanticType", + "pf"=>"prefLabelSuggest^50", + } + resp = TermSearch.search("Cutaneous Melanoma", params) + assert_equal(2, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + + # test NOT filtering out unicode characters + resp = TermSearch.search("肿瘤", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[1].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_exact_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelExact", + } + resp = TermSearch.search("Melanoma", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[0].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_suggest_edge_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggestEdge", + } + resp = TermSearch.search("Melanoma with", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + + resp = TermSearch.search("Melanoma", params) + assert_equal(2, resp["response"]["numFound"]) + assert_equal @terms[0].prefLabel, resp["response"]["docs"][0]["prefLabel"] + end + + def test_search_suggest_ngram_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggestNgram", + } + resp = TermSearch.search("cutaneous", params) + assert_equal(1, resp["response"]["numFound"]) + assert_equal @terms[2].prefLabel, resp["response"]["docs"][0]["prefLabel"] + + resp = TermSearch.search("eous", params) + assert_equal(0, resp["response"]["numFound"]) + end + + def test_search_suggest_filter + TermSearch.indexClear + @terms[0].index + @terms[1].index + @terms[2].index + TermSearch.indexCommit + params = {"defType"=>"edismax", + "stopwords"=>"true", + "lowercaseOperators"=>"true", + "qf"=>"prefLabelSuggest", + } + resp = TermSearch.search("cutaneous test with Neoplasm Melanoma", params) + assert_equal(3, resp["response"]["numFound"]) + + + resp = TermSearch.search("mel", params) + assert_equal(0, resp["response"]["numFound"]) + end + def test_unindex TermSearch.indexClear() @terms[1].index() @@ -87,14 +278,14 @@ def test_unindexByQuery @terms[1].index() TermSearch.indexCommit() resp = TermSearch.search(@terms[1].prefLabel) - assert_equal(1, resp["response"]["docs"].length) + assert_equal 1, resp["response"]["docs"].length query = "submissionAcronym:" + @terms[1].submissionAcronym TermSearch.unindexByQuery(query) TermSearch.indexCommit() resp = TermSearch.search(@terms[1].prefLabel) - assert_equal(0, resp["response"]["docs"].length) + assert_equal 0, resp["response"]["docs"].length end def test_index @@ -102,7 +293,7 @@ def test_index @terms[0].index() TermSearch.indexCommit() resp = TermSearch.search(@terms[0].prefLabel) - assert_equal(1, resp["response"]["docs"].length) + assert_equal 1, resp["response"]["docs"].length assert_equal @terms[0].prefLabel, resp["response"]["docs"][0]["prefLabel"] end @@ -111,7 +302,7 @@ def test_indexBatch TermSearch.indexBatch(@terms) TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal(2, resp["response"]["docs"].length) + assert_equal @terms.size, resp["response"]["docs"].length end def test_unindexBatch @@ -119,20 +310,83 @@ def test_unindexBatch TermSearch.indexBatch(@terms) TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal(2, resp["response"]["docs"].length) + assert_equal @terms.size, resp["response"]["docs"].length TermSearch.unindexBatch(@terms) TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal(0, resp["response"]["docs"].length) + assert_equal 0, resp["response"]["docs"].length end def test_indexClear TermSearch.indexClear() TermSearch.indexCommit() resp = TermSearch.search("*:*") - assert_equal(0, resp["response"]["docs"].length) + assert_equal 0, resp["response"]["docs"].length + end + + def test_index_on_save_delete + TermSearch2.find("test").first&.delete + TermSearch3.find("test2").first&.delete + + term = TermSearch2.new(prefLabel: "test", + submissionId: 1, + definition: "definition of test", + synonym: ["synonym1", "synonym2"], + submissionAcronym: "test", + private: true + ) + + term2 = TermSearch3.new(prefLabel: "test2", + submissionId: 1, + definition: "definition of test2", + synonym: ["synonym1", "synonym2"], + submissionAcronym: "test", + private: true, + object: TermSearch.new(prefLabel: "test", submissionAcronym: 'acronym', submissionId: 1 ).save, + object_list: [TermSearch.new(prefLabel: "test2",submissionAcronym: 'acronym2', submissionId: 2).save, + TermSearch.new(prefLabel: "test3", submissionAcronym: 'acronym3', submissionId: 3).save] + ) + + term.save + term2.save + + # set as not indexed in model definition + refute_includes TermSearch2.search_client.fetch_all_fields.map{|f| f["name"]}, "private_b" + refute_includes TermSearch2.search_client.fetch_all_fields.map{|f| f["name"]}, "private_b" + + + indexed_term = TermSearch2.search("id:#{term.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + indexed_term2 = TermSearch3.search("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + term.indexable_object.each do |k, v| + assert_equal v, indexed_term[k.to_s] + end + + term2.indexable_object.each do |k, v| + assert_equal v, indexed_term2[k.to_s] + end + + term2.definition = "new definition of test2" + term2.synonym = ["new synonym1", "new synonym2"] + term2.save + + indexed_term2 = TermSearch3.search("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + term2.indexable_object.each do |k, v| + assert_equal v, indexed_term2[k.to_s] + end + + term2.delete + term.delete + + indexed_term = TermSearch2.submit_search_query("id:#{term.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + indexed_term2 = TermSearch3.submit_search_query("id:#{term2.id.to_s.gsub(":", "\\:")}")["response"]["docs"].first + + assert_nil indexed_term + assert_nil indexed_term2 + end end -end #module +end diff --git a/test/test_update_callbacks.rb b/test/test_update_callbacks.rb new file mode 100644 index 00000000..bef38a68 --- /dev/null +++ b/test/test_update_callbacks.rb @@ -0,0 +1,53 @@ +require_relative 'test_case' + + +require_relative 'models' + +class TestUpdateCallBack < Goo::Base::Resource + model :update_callback_model, name_with: :code + attribute :code, enforce: [:string, :existence] + attribute :name, enforce: [:string, :existence] + attribute :first_name, onUpdate: :update_name + attribute :last_name, onUpdate: :update_name + + + def update_name(inst, attr) + self.name = self.first_name + self.last_name + end +end + +class TestUpdateCallBacks < MiniTest::Unit::TestCase + + def self.before_suite + GooTestData.delete_all [TestUpdateCallBack] + end + + def self.after_suite + GooTestData.delete_all [TestUpdateCallBack] + end + + + def test_update_callback + p = TestUpdateCallBack.new + p.code = "1" + p.name = "name" + p.first_name = "first_name" + p.last_name = "last_name" + + assert p.valid? + p.save + + p.bring_remaining + + assert_equal p.first_name + p.last_name, p.name + + p.last_name = "last_name2" + p.save + + p.bring_remaining + assert_equal "last_name2", p.last_name + assert_equal p.first_name + p.last_name, p.name + end + +end + diff --git a/test/test_url_validator.rb b/test/test_url_validator.rb new file mode 100644 index 00000000..ac7af9a3 --- /dev/null +++ b/test/test_url_validator.rb @@ -0,0 +1,63 @@ +require_relative 'test_case' +require 'goo/validators/validator' +require 'goo/validators/implementations/data_type' +require 'rdf' + +class UrlTestModel < Goo::Base::Resource + model :url_test_model, name_with: :name + attribute :url, enforce: %i[url] + attribute :urls, enforce: %i[list url] +end + +class UrlValidatorTest < Minitest::Unit::TestCase + def test_url_scalar + u = UrlTestModel.new + u.url = RDF::URI.new('https://example.com/path?x=1') + assert u.valid?, "expected https URL to be valid, got errors: #{u.errors.inspect}" + + u.url = [RDF::URI.new('https://example.com/path?x=1')] + refute u.valid?, "expected to reject array, got errors: #{u.errors.inspect}" + assert u.errors[:url][:no_list], "errors: #{u.errors.inspect}" + end + + def test_url_scalar_rejects_non_http_schemes + [ + '', 'http://', 'wrong/uri', 'mailto:user@nodomain.org', 'ftp://test.com/', + 'urn:isbn:123456', 'ssh://root@localhost:22', 'file:///etc/passwd', + 'http://', 'http://[::gggg]', + '//example.org/path', + "https://example.com/too_long_url/#{'a' * 2050}" + ].each do |bad| + u = UrlTestModel.new + u.url = RDF::URI.new(bad) + refute u.valid?, "expected invalid for #{bad.inspect}" + assert u.errors[:url][:url], "expected :url error key for #{bad.inspect}" + end + end + + def test_url_list + u = UrlTestModel.new + u.urls = [RDF::URI.new('http://example.com/'), + RDF::URI.new('https://example2.com/ok')] + assert u.valid?, "expected valid list of URLs, got: #{u.errors.inspect}" + + u.urls = [RDF::URI.new('https://example.com/')] + assert u.valid? + + u.urls = RDF::URI.new('http://example.com/') + refute u.valid? + assert u.errors[:urls] + end + + def test_url_list_must_all_be_valid + u = UrlTestModel.new + u.urls = [RDF::URI.new('https://ok.example'), + RDF::URI.new('mailto:bad@example.org')] + refute u.valid? + assert u.errors[:urls][:url] + + u.urls = [RDF::URI.new('https://ok.example'), true] + refute u.valid? + assert u.errors[:urls][:url] + end +end \ No newline at end of file diff --git a/test/test_validators.rb b/test/test_validators.rb new file mode 100644 index 00000000..d73fee76 --- /dev/null +++ b/test/test_validators.rb @@ -0,0 +1,504 @@ +require_relative 'test_case' + +require_relative 'models' + +class Person < Goo::Base::Resource + model :person_model_validators, name_with: :name + attribute :name, enforce: [:string, :existence] + attribute :username, enforce: [:string, :existence, :username] + attribute :last_name, enforce: [:string] + attribute :multiple_values, enforce: [ :list, :integer] + attribute :one_number, enforce: [ :integer ] + attribute :birth_date, enforce: [ :date_time ] + attribute :male, enforce: [:boolean] + attribute :social, enforce: [:uri] + attribute :email, enforce: [:email] + attribute :socials, enforce: [:uri, :list] + attribute :weight, enforce: [:float] + attribute :friends, enforce: [Person, :list] +end + +class SafeTextTestModel < Goo::Base::Resource + model :safe_text_test_model, name_with: :name + attribute :first_name, enforce: [:safe_text_5, :existence] + attribute :last_name, enforce: [:safe_text_8, :existence] + attribute :description, enforce: [:safe_text, :existence] +end + +class RangeTestModel < Goo::Base::Resource + model :range_test_model, name_with: :name + attribute :name, enforce: [:string, :existence, :min_3, :max_5] + attribute :multiple_values, enforce: [ :list, :integer, :min_3, :max_5 ] + attribute :one_number, enforce: [ :integer, :min_3, :max_5] + attribute :weight, enforce: [:float, :min_3, :max_5] +end + +class SymmetricTestModel < Goo::Base::Resource + model :symmetric_test_model, name_with: :name + attribute :name, enforce: [:unique, :existence] + attribute :friend, enforce: [SymmetricTestModel, :symmetric] + attribute :friends, enforce: [SymmetricTestModel, :symmetric, :list] +end + +class DistinctOfTestModel < Goo::Base::Resource + model :distinct_of_test_model, name_with: :name + attribute :name, enforce: [:unique, :existence, :string] + attribute :last_name, enforce: [:distinct_of_name, :string] + attribute :names, enforce: [:list, :string] + attribute :last_names, enforce: [:list, :distinct_of_names, :string] +end + +class SuperiorToTestModel < Goo::Base::Resource + model :superior_to_test_model, name_with: :name + attribute :name, enforce: [:unique, :existence, :string] + attribute :birth_date, enforce: [:date_time] + attribute :death_date, enforce: [:superior_equal_to_birth_date, :date_time] +end + +class InverseOfTestModel < Goo::Base::Resource + model :inverse_test_model_one, name_with: :name + attribute :name, enforce: [:unique, :existence, :string] + attribute :state, enforce: [InverseOfTestModel] + attribute :city, enforce: [:inverse_of_state, InverseOfTestModel] + attribute :states, enforce: [InverseOfTestModel, :list] + attribute :cities, enforce: [:inverse_of_states, InverseOfTestModel, :list] +end + + +class ProcValidatorsTestModel < Goo::Base::Resource + model :proc_validator_test_model, name_with: :name + attribute :name, enforce: [:unique, :equal_to_test] + attribute :last_name, enforce: [:unique, ->(inst, attr) { equal_to_test_2(inst, attr)}] + + + def self.equal_to_test_2(inst, attr) + value = inst.send(attr) + + return nil if value && value.eql?('test 2') + + [:equal_to_test_2, "#{attr} need to be equal to `test 2`"] + end + + def equal_to_test(inst, attr) + value = inst.send(attr) + + return nil if value && value.eql?('test') + + [:equal_to_test, "#{attr} need to be equal to `test`"] + end +end + +class TestValidators < MiniTest::Unit::TestCase + + def self.before_suite + begin + GooTestData.create_test_case_data + rescue Exception => e + puts e.message + end + end + + def self.after_suite + GooTestData.delete_test_case_data + GooTestData.delete_all [SymmetricTestModel, InverseOfTestModel] + end + + def test_unique_validator + + s = Student.new + s.birth_date = DateTime.parse('1978-01-01') + + s.name = "Susan" + + refute s.valid? + + s.name = "new" + + assert s.valid? + end + + def test_username_validator + p = Person.new + p.name = "Susan" + p.username = "goodusername" + assert p.valid? + + p.username = "good_username" + assert p.valid? + + p.username = "good.username" + assert p.valid? + + p.username = "bad-username" + refute p.valid? + + p.username = "1badusername" + refute p.valid? + + p.username = "badusername with spaces" + refute p.valid? + + p.username = "\\x3csVg/\\x3e\">" + refute p.valid? + end + + def test_safe_text_validator + m = SafeTextTestModel.new + m.first_name = 'Susan' + m.last_name = 'Johnson' + m.description = 'The name Susan carries a rich history and evokes a sense of grace, warmth, and intelligence. While its popularity has somewhat declined, it remains a classic name with a positive reputation and a strong legacy of accomplished individuals associated with it.' + assert m.valid? + + m.first_name = 'Michael' + refute m.valid? + assert_equal 1, m.errors.keys.length + assert m.errors[:first_name][:safe_text_5].include?('and must not exceed 5 characters') + + m.first_name = 'Joe' + m.description = 'The name Susan 🌍 carries a rich history' + refute m.valid? + assert_equal 1, m.errors.keys.length + assert_equal :description, m.errors.keys[0] + + m.description = "I am a valid description" + assert m.valid? + + m.description = "临床表现" + assert m.valid? + + m.description = "This string contains\na newline" + refute m.valid? + + m.description = "This string has a tab\tcharacter" + refute m.valid? + + m.description = "Price is < than $1!" + refute m.valid? + + m.description = "This has\u200Bhidden content" + refute m.valid? + + m.description = "Normal text\u202Eevil.com" + refute m.valid? + end + + def test_existence_validator + s = Student.new + + refute s.valid? + + assert s.errors[:name][:existence] + assert s.errors[:birth_date][:existence] + + + s.name = '' + s.birth_date = '' + assert s.errors[:name][:existence] + assert s.errors[:birth_date][:existence] + + + s.name = 'new' + s.birth_date = DateTime.parse('1978-01-01') + + assert s.valid? + end + + def test_datatype_validators + p = Person.new + p.name = 'test' + p.username = 'test_username' + #nil values are valid + assert p.valid? + + p.last_name = false + p.multiple_values = "hello" + p.one_number = "hello" + p.birth_date = 100 + p.male = "ok" + p.social = 100 + p.socials = [100] + p.weight = 100 + p.email = "test@test" + #wrong types are not valid + refute p.valid? + assert p.errors[:last_name][:string] + assert p.errors[:multiple_values][:list] + assert p.errors[:multiple_values][:integer] + assert p.errors[:one_number][:integer] + assert p.errors[:birth_date][:date_time] + assert p.errors[:male][:boolean] + assert p.errors[:social][:uri] + assert p.errors[:email][:email] + + p.last_name = "hello" + p.multiple_values = [22,11] + p.one_number = 12 + p.birth_date = DateTime.parse('1978-01-01') + p.male = true + p.social = RDF::URI.new('https://test.com/') + p.socials = [RDF::URI.new('https://test.com/'), RDF::URI.new('https://test.com/')] + p.weight = 100.0 + p.email = "test@test.hi.com" + #good types are valid + assert p.valid? + end + + def test_uri_datatype_validator + p = Person.new + p.name = 'test' + p.username = 'test_username' + assert p.valid? + + p.social = RDF::URI.new('') #empty uri + refute p.valid? + + p.social = RDF::URI.new('wrong/uri') + refute p.valid? + + p.social = RDF::URI.new('https://test.com/') + assert p.valid? + end + + def test_object_type_validator + p = Person.new + p.name = 'test' + p.username = 'test_username' + p.friends = [1] + + refute p.valid? + + new_person = Person.new + p.friends = [new_person] + + refute p.valid? + + new_person.persistent = true + p.friends = [new_person] + + assert p.valid? + end + + def test_value_range_validator + p = RangeTestModel.new + + p.name = "h" + p.multiple_values = [22,11] + p.one_number = 1 + p.weight = 1.1 + + refute p.valid? + assert p.errors[:name][:min] + assert p.errors[:multiple_values][:min] + assert p.errors[:one_number][:min] + assert p.errors[:weight][:min] + + p.name = "hello hello" + p.multiple_values = [22,11,11,33,44, 55, 66] + p.one_number = 12 + p.weight = 12.1 + + refute p.valid? + assert p.errors[:name][:max] + assert p.errors[:multiple_values][:max] + assert p.errors[:one_number][:max] + assert p.errors[:weight][:max] + + p.name = "hello" + p.multiple_values = [22,11,11,3] + p.one_number = 4 + p.weight = 3.1 + + assert p.valid? + + end + + def test_symmetric_validator_no_list + p1 = SymmetricTestModel.new + p2 = SymmetricTestModel.new + p3 = SymmetricTestModel.new + p1.name = "p1" + p2.name = "p2" + p3.name = "p3" + + p2.save + p3.save + + p1.friend = p2 + + refute p1.valid? + assert p1.errors[:friend][:symmetric] + + p3.friend = p1 + + refute p1.valid? + + p2.friend = p1 + p1.friend = p2 + + assert p1.valid? + + p1.save + + assert p2.valid? + GooTestData.delete_all [SymmetricTestModel] + end + + def test_symmetric_validator_list + p1 = SymmetricTestModel.new + p2 = SymmetricTestModel.new + p3 = SymmetricTestModel.new + p4 = SymmetricTestModel.new + p1.name = "p1" + p2.name = "p2" + p3.name = "p3" + p4.name = "p4" + + p2.save + p3.save + p4.save + + p1.friends = [p2, p3] + + refute p1.valid? + assert p1.errors[:friends][:symmetric] + + p2.friends = [p1, p3, p4] + p3.friends = [p2] + p4.friends = [p2] + + refute p1.valid? + refute p2.valid? + + + p3.friends = [p2, p1] + + assert p1.valid? + p1.save + + assert p3.valid? + p3.save + + + assert p2.valid? + + p2.save + + assert p4.valid? + GooTestData.delete_all [SymmetricTestModel] + end + + def test_distinct_of_validator + p = DistinctOfTestModel.new + p.name = "p1" + p.last_name = "p1" + p.names = ["p1", "p2"] + p.last_names = ["p1", "p2"] + + + refute p.valid? + + p.last_name = "last name" + p.last_names = ["last name 1", "last name 2"] + + assert p.valid? + + p.last_name = "last name" + p.last_names = ["last name 1", "p2"] + + refute p.valid? + + p.last_name = "" + p.last_names = [] + + assert p.valid? + end + + def test_superior_equal_to_validator + p = SuperiorToTestModel.new + p.name = "p" + p.birth_date = DateTime.parse('1998-12-02') + p.death_date = DateTime.parse('1995-12-02') + + refute p.valid? + assert p.errors[:death_date][:superior_equal_to_birth_date] + + p.death_date = DateTime.parse('2023-12-02') + + assert p.valid? + + p.birth_date = nil + + assert p.valid? + end + + def test_inverse_of_validator_no_list + GooTestData.delete_all [InverseOfTestModel] + p1 = InverseOfTestModel.new + p2 = InverseOfTestModel.new + + p1.name = 'p1' + p2.name = 'p2' + + + p2.save + + p1.city = p2 + + refute p1.valid? + assert p1.errors[:city][:inverse_of_state] + + + p2.state = p1 + + assert p1.valid? + + end + + def test_inverse_of_validator_list + GooTestData.delete_all [InverseOfTestModel] + p1 = InverseOfTestModel.new + p2 = InverseOfTestModel.new + p3 = InverseOfTestModel.new + p4 = InverseOfTestModel.new + + p1.name = 'p1' + p2.name = 'p2' + p3.name = 'p3' + p4.name = 'p4' + + p2.save + p3.save + + p1.cities = [p2,p3] + + refute p1.valid? + assert p1.errors[:cities][:inverse_of_states] + + p2.states = [p1, p4] + p3.states = [p2, p4] + + refute p1.valid? + assert p1.errors[:cities][:inverse_of_states] + + p3.states = [p2, p4, p1] + + assert p1.valid? + + end + + + def test_proc_validators + p = ProcValidatorsTestModel.new + p.name = "hi" + p.last_name = "hi" + + refute p.valid? + assert p.errors[:name][:equal_to_test] + assert p.errors[:last_name][:equal_to_test_2] + + p.name = "test" + p.last_name = "test 2" + + assert p.valid? + end +end diff --git a/test/test_where.rb b/test/test_where.rb index 1d08a223..a95131b6 100644 --- a/test/test_where.rb +++ b/test/test_where.rb @@ -1,7 +1,4 @@ require_relative 'test_case' - -GooTest.configure_goo - require_relative 'models' class TestWhere < MiniTest::Unit::TestCase @@ -14,7 +11,7 @@ def self.before_suite begin GooTestData.create_test_case_data rescue Exception => e - binding.pry + puts e.message end end @@ -23,20 +20,20 @@ def self.after_suite end def test_where_simple - assert University.range(:programs) == Program + assert_equal Program, University.range(:programs) st = University.where(name: "Stanford") - assert st.length == 1 + assert_equal 1, st.length st = st.first - assert st.instance_of?(University) + assert_instance_of University, st st.bring(programs: [:credits]) - assert st.programs.length == 3 + assert_equal 3, st.programs.length st.programs.each do |p| - assert_instance_of Fixnum, p.credits + assert_instance_of Integer, p.credits end - #nothing is loaded + # No attributes loaded st.class.attributes.each do |attr| assert_raises Goo::Base::AttributeNotLoaded do st.send("#{attr}") @@ -44,33 +41,36 @@ def test_where_simple end st = University.where(name: "Stanford").include(University.attributes).all - assert st.length == 1 + assert_equal 1, st.length st = st.first - assert assert st.instance_of?(University) - assert st.name == "Stanford" + assert_instance_of University, st + assert_equal "Stanford", st.name assert_raises Goo::Base::AttributeNotLoaded do st.programs end - # - #all includes inverse + + # All clause (includes inverse) st = University.where(name: "Stanford").include(University.attributes(:all)).all - assert st.length == 1 + assert_equal 1, st.length st = st.first - assert st.instance_of?(University) - assert st.name == "Stanford" - assert st.programs.length == 3 - #programs here are not loaded + assert_instance_of University, st + assert_equal "Stanford", st.name + assert_equal 3, st.programs.length + + # Programs aren't loaded pr = st.programs[0] pr.class.attributes.each do |attr| assert_raises Goo::Base::AttributeNotLoaded do pr.send("#{attr}") end end - program_ids = ["http://example.org/program/Stanford/BioInformatics", - "http://example.org/program/Stanford/CompSci", - "http://example.org/program/Stanford/Medicine"] - assert st.programs.map { |x| x.id.to_s }.sort == program_ids + program_ids = [ + "http://example.org/program/Stanford/BioInformatics", + "http://example.org/program/Stanford/CompSci", + "http://example.org/program/Stanford/Medicine" + ] + assert_equal program_ids, st.programs.map { |x| x.id.to_s }.sort end def test_all @@ -101,7 +101,7 @@ def test_where_2levels programs = Program.where(name: "BioInformatics", university: [ address: [ country: "UK" ]]).all assert programs.length == 1 assert programs.first.id.to_s["Southampton/BioInformatics"] - + #any program from universities in the US programs = Program.where(university: [ address: [ country: "US" ]]).include([:name]).all assert programs.length == 3 @@ -118,15 +118,15 @@ def test_where_2levels_inverse #equivalent unis = University.where(address: [country: "US"]) - .and(programs: [category: [code: "Biology"]]).all + .and(programs: [category: [code: "Biology"]]).all assert unis.length == 1 assert unis.first.id.to_s == "http://goo.org/default/university/Stanford" end def test_embed_include programs = Program.where.include(:name) - .include(university: [:name]) - .include(category: [:code]).all + .include(university: [:name]) + .include(category: [:code]).all assert programs.length == 9 programs.each do |p| @@ -177,7 +177,7 @@ def test_iterative_include_in_place #two levels unis = University.where.all unis_return = University.where.models(unis) - .include(programs: [:name, students: [:name]]).to_a + .include(programs: [:name, students: [:name]]).to_a assert unis_return.object_id == unis.object_id return_object_id = unis.map { |x| x.object_id }.uniq.sort unis_object_id = unis.map { |x| x.object_id }.uniq.sort @@ -259,6 +259,38 @@ def test_embed_two_levels end end + def test_fetch_remaining + students = Student.where(enrolled:RDF::URI.new("http://example.org/program/Stanford/BioInformatics")) + .include(:name, :birth_date, enrolled: [:name]).all + + + s = students.select { |x| x.name['Daniel'] }.first + refute_nil s + assert_equal 2, s.enrolled.size + end + + def test_paging_with_filter_order + skip('pagination with filter and order does not work in 4s') if Goo.backend_4s? + + f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03') + total_count = Student.where.filter(f).count + page_1 = Student.where.include(:name, :birth_date).page(1, total_count - 1).filter(f).order_by(name: :asc).to_a + refute_empty page_1 + assert page_1.next? + page_2 = Student.where.include(:name, :birth_date).page(page_1.next_page, total_count - 1).filter(f).order_by(name: :asc).to_a + + + refute_empty page_2 + assert_equal total_count, page_1.size + page_2.size + end + + def test_two_level_include + programs = Program.where.include(:name).all + r = Program.where.models(programs).include(students: [:name]).all + r.each do |p| + refute_nil p.students + end + end def test_unique_object_references @@ -319,7 +351,7 @@ def test_unique_object_references def test_complex_include #Students in a university by name students = Student.where(enrolled: [university: [name: "Stanford"]]) - .include(:name) + .include(:name) .include(enrolled: [:name, university: [ :address ]]).all assert students.map { |x| x.name }.sort == ["Daniel","John","Susan"] @@ -329,7 +361,7 @@ def test_complex_include assert_instance_of University, p.university assert_instance_of Array, p.university.addresses assert_instance_of Address, p.university.addresses.first - assert_raises Goo::Base::AttributeNotLoaded do + assert_raises Goo::Base::AttributeNotLoaded do p.university.addresses.first.country end end @@ -389,23 +421,23 @@ def test_where_union_pattern def test_where_direct_attributes st = Student.where(name: "Daniel") - .or(name: "Louis") - .or(name: "Lee") - .or(name: "John").all + .or(name: "Louis") + .or(name: "Lee") + .or(name: "John").all assert st.length == 4 st = Student.where(name: "Daniel") - .and(name: "John").all + .and(name: "John").all assert st.length == 0 st = Student.where(name: "Daniel") - .and(birth_date: DateTime.parse('1978-01-04')).all + .and(birth_date: DateTime.parse('1978-01-04')).all assert st.length == 1 assert st.first.id.to_s["Daniel"] st = Student.where(name: "Daniel") - .or(name: "Louis") - .and(birth_date: DateTime.parse('1978-01-04')) + .or(name: "Louis") + .and(birth_date: DateTime.parse('1978-01-04')) assert st.length == 1 assert st.first.id.to_s["Daniel"] @@ -441,8 +473,8 @@ def test_combine_where_patterns_with_include st.each do |p| assert (p.name == "Susan" || p.name == "Daniel") assert Array, p.enrolled - assert (p.name == "Susan" && p.enrolled.length == 1) || - (p.name == "Daniel" && p.enrolled.length == 2) + assert (p.name == "Susan" && p.enrolled.length == 1) || + (p.name == "Daniel" && p.enrolled.length == 2) assert String, p.enrolled.first.university.address.first.country end end @@ -454,31 +486,31 @@ def test_filter f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03') st = Student.where.filter(f).all assert st.map { |x| x.id.to_s }.sort == ["http://goo.org/default/student/Daniel", - "http://goo.org/default/student/Lee", - "http://goo.org/default/student/Louis", - "http://goo.org/default/student/Robert"] + "http://goo.org/default/student/Lee", + "http://goo.org/default/student/Louis", + "http://goo.org/default/student/Robert"] f = (Goo::Filter.new(:birth_date) <= DateTime.parse('1978-01-01')) .or(Goo::Filter.new(:birth_date) >= DateTime.parse('1978-01-07')) st = Student.where.filter(f).all assert st.map { |x| x.id.to_s }.sort == [ - "http://goo.org/default/student/Robert", - "http://goo.org/default/student/Susan"] + "http://goo.org/default/student/Robert", + "http://goo.org/default/student/Susan"] f = (Goo::Filter.new(:birth_date) <= DateTime.parse('1978-01-01')) .or(Goo::Filter.new(:name) == "Daniel") st = Student.where.filter(f).all assert st.map { |x| x.id.to_s }.sort == [ - "http://goo.org/default/student/Daniel", - "http://goo.org/default/student/Susan"] + "http://goo.org/default/student/Daniel", + "http://goo.org/default/student/Susan"] f = (Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-02')) .and(Goo::Filter.new(:birth_date) < DateTime.parse('1978-01-06')) st = Student.where.filter(f).all assert st.map { |x| x.id.to_s }.sort == [ - "http://goo.org/default/student/Daniel", - "http://goo.org/default/student/Louis", - "http://goo.org/default/student/Tim"] + "http://goo.org/default/student/Daniel", + "http://goo.org/default/student/Louis", + "http://goo.org/default/student/Tim"] f = Goo::Filter.new(enrolled: [ :credits ]) > 8 @@ -488,14 +520,20 @@ def test_filter #students without awards f = Goo::Filter.new(:awards).unbound st = Student.where.filter(f) - .include(:name) - .all + .include(:name) + .all assert st.map { |x| x.name }.sort == ["John","Tim","Louis","Lee","Robert"].sort #unbound on some non existing property f = Goo::Filter.new(enrolled: [ :xxx ]).unbound st = Student.where.filter(f).all assert st.length == 7 + + f = Goo::Filter.new(:name).regex("n") # will find all students that contains "n" in there name + st = Student.where.filter(f).include(:name).all # return "John" , "Daniel" and "Susan" + + assert_equal 3, st.length + assert_equal ["John","Daniel","Susan"].sort, st.map { |x| x.name }.sort end def test_aggregated @@ -518,7 +556,7 @@ def test_aggregated sts = Student.where.include(:name).aggregate(:count, :enrolled).all sts.each do |st| assert (st.name == "Daniel" && st.aggregates.first.value == 2) || - st.aggregates.first.value == 1 + st.aggregates.first.value == 1 end #students enrolled in more than 1 program and get the programs name @@ -591,4 +629,15 @@ def test_include_inverse_with_find end end + def test_complex_order_by + u = University.where.include(address: [:country]).order_by(address: {country: :asc}).all + countries = u.map {|x| x.address.map{|a| a.country}}.flatten + assert_equal countries.sort, countries + + + u = University.where.include(address: [:country]).order_by(address: {country: :desc}).all + countries = u.map {|x| x.address.map{|a| a.country}}.flatten + assert_equal countries.sort{|a,b| b<=>a }, countries + end + end