Update to new plugin format

.readthedocs.yml

@@ -1,3 +1,10 @@
+version: 2
+
+build:
+  os: "ubuntu-20.04"
+  tools:
+   python: "3.7"
+
 python:
    install:
       - requirements: docs/requirements.txt

.travis.yml

@@ -1,4 +1,9 @@
 language: python
+env:
+  global:
+    - PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow"
+    - MODIFIES_PLUGIN="False"
+    - PLUGIN_ONLY="False"
 matrix:
   include:
   - name: 3.7 public
@@ -16,13 +21,28 @@ before_install:
 - pip install pytest
 # download large files
 - pip install awscli
-- bash test_setup.sh
 install:
-- pip install .
+- pip install -e ".[test]"
+# install conda for plugin runner
+- wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+- bash miniconda.sh -b -p $HOME/miniconda
+- source "$HOME/miniconda/etc/profile.d/conda.sh"
+- hash -r
+- conda config --set always_yes yes --set changeps1 no
+- conda update -q conda
+- conda info -a
 - pip list # list installed package versions
 script:
-- if [ "$PRIVATE_ACCESS" = 1 ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then pytest -m "not requires_gpu and not memory_intense and not slow and not travis_slow"; fi
-- if [ "$PRIVATE_ACCESS" != 1 ]; then pytest -m "not requires_gpu and not memory_intense and not private_access and not slow and not travis_slow"; fi
+  # if ONLY plugin files changed, ONLY run tests for those plugins; otherwise, run full test suite
+  - |
+    if [ ! -z "$TRAVIS_PULL_REQUEST_BRANCH" ]; then 
+      CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) &&
+      TESTING_NEEDED=$( python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_testing_info; get_testing_info(\"${CHANGED_FILES}\", 'brainscore_vision')" ) && 
+      read MODIFIES_PLUGIN PLUGIN_ONLY <<< $TESTING_NEEDED && echo MODIFIES_PLUGIN: $MODIFIES_PLUGIN && echo PLUGIN_ONLY: $PLUGIN_ONLY; 
+    fi
+  - if [ "$MODIFIES_PLUGIN" = "True" ]; then python -c "from brainscore_core.plugin_management.parse_plugin_changes import run_changed_plugin_tests; run_changed_plugin_tests(\"${CHANGED_FILES}\", 'brainscore_vision')"; fi
+  - if [ "$PRIVATE_ACCESS" = 1 ] && [ -n "${GITHUB_TOKEN}" ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "private_access and $PYTEST_SETTINGS"; fi
+  - if [ "$PRIVATE_ACCESS" != 1 ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "not private_access and $PYTEST_SETTINGS" --ignore "tests/test_submission"; fi
 
 notifications:
   slack:

MANIFEST.in

@@ -1,6 +1,8 @@
 include LICENSE
 include README.md
-include brainscore/lookup.csv
+include brainscore_vision/lookup.csv
+include brainscore_vision/model_helpers/brain_transformation/imagenet_classes.txt
+include brainscore_vision/model_helpers/check_submission/images/*.png
 
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

README.md

@@ -6,10 +6,10 @@ on their match to brain measurements in primate vision.
 The intent of Brain-Score is to adopt many (ideally all) the experimental benchmarks in the field
 for the purpose of model testing, falsification, and comparison.
 To that end, Brain-Score operationalizes experimental data into quantitative benchmarks 
-that any model candidate following the [`BrainModel`](brainscore/model_interface.py) interface can be scored on.
+that any model candidate following the [`BrainModel`](brainscore_vision/model_interface.py) interface can be scored on.
 
 See the [Documentation](https://brain-score.readthedocs.io) for more details 
-and the [Tutorial](https://brain-score.readthedocs.io/en/latest/modules/tutorial.html) 
+and the [Tutorial](https://brain-score.readthedocs.io/en/latest/modules/model_tutorial.html) 
 and [Examples](https://github.com/brain-score/candidate_models/blob/master/examples/score-model.ipynb)
 for submitting a model to Brain-Score.
 
@@ -19,28 +19,29 @@ To contribute, please [send in a pull request](https://github.com/brain-score/br
 
 ## Local installation
 
-You will need Python >= 3.7 and pip >= 18.1.
+You will need Python = 3.7 and pip >= 18.1.
 Note that you can only access public benchmarks when running locally.
 To score a model on all benchmarks, submit it via the [brain-score.org website](http://www.brain-score.org).
 
 `pip install git+https://github.com/brain-score/brain-score`
 
 Score a model on a public benchmark:
+
 ```python
-from brainscore.benchmarks import public_benchmark_pool
+from brainscore_vision.benchmarks import public_benchmark_pool
 
 benchmark = public_benchmark_pool['dicarlo.MajajHong2015public.IT-pls']
 model = my_model()
 score = benchmark(model)
-#>  <xarray.Score (aggregation: 2)>
-#>  array([0.32641998, 0.0207475])
-#>  Coordinates:
-#>    * aggregation  (aggregation) <U6 'center' 'error'
-#>  Attributes:
-#>      raw:                   <xarray.Score (aggregation: 2)>\narray([0.4278365 ...
-#>      ceiling:               <xarray.Score (aggregation: 2)>\narray([0.7488407 ...
-#>      model_identifier:      my-model
-#>      benchmark_identifier:  dicarlo.MajajHong2015public.IT-pls
+# >  <xarray.Score (aggregation: 2)>
+# >  array([0.32641998, 0.0207475])
+# >  Coordinates:
+# >    * aggregation  (aggregation) <U6 'center' 'error'
+# >  Attributes:
+# >      raw:                   <xarray.Score (aggregation: 2)>\narray([0.4278365 ...
+# >      ceiling:               <xarray.Score (aggregation: 2)>\narray([0.7488407 ...
+# >      model_identifier:      my-model
+# >      benchmark_identifier:  dicarlo.MajajHong2015public.IT-pls
 ```
 
 Some steps may take minutes because data has to be downloaded during first-time use.