Jupyter book 2 migration

.actrc

@@ -0,0 +1,3 @@
+-P ubuntu-latest=catthehacker/ubuntu:act-latest
+--container-architecture linux/amd64
+--artifact-server-path /tmp/artifacts

.github/actions/setup-ci-tools/action.yml

@@ -0,0 +1,58 @@
+name: 'Setup nmaci CI Tools'
+description: 'Download and install Neuromatch CI tools'
+
+inputs:
+  branch:
+    description: 'nmaci branch to use'
+    required: false
+    default: 'main'
+  commit-message:
+    description: 'Commit message to parse for branch override'
+    required: false
+    default: ''
+
+outputs:
+  nmaci-branch:
+    description: 'nmaci branch used'
+    value: ${{ steps.detect-branch.outputs.branch }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Detect nmaci branch
+      id: detect-branch
+      shell: bash
+      run: |
+        BRANCH="${{ inputs.branch }}"
+        if [ -n "${{ inputs.commit-message }}" ]; then
+          OVERRIDE=$(python3 -c "import os, re; m = re.search(r'nmaci:([\w-]+)', '${{ inputs.commit-message }}'); print(m.group(1) if m else '')")
+          if [ -n "$OVERRIDE" ]; then
+            BRANCH="$OVERRIDE"
+          fi
+        fi
+        echo "branch=$BRANCH" >> $GITHUB_OUTPUT
+        echo "Using nmaci branch: $BRANCH"
+
+    - name: Cache nmaci tools
+      id: cache-nmaci
+      uses: actions/cache@v3
+      with:
+        path: ci/
+        key: nmaci-${{ steps.detect-branch.outputs.branch }}-${{ hashFiles('.github/workflows/*.yml', '.github/workflows/*.yaml') }}
+        restore-keys: nmaci-${{ steps.detect-branch.outputs.branch }}-
+
+    - name: Download nmaci tools
+      if: steps.cache-nmaci.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        BRANCH=${{ steps.detect-branch.outputs.branch }}
+        wget -q https://github.com/neuromatch/nmaci/archive/refs/heads/$BRANCH.tar.gz
+        tar -xzf $BRANCH.tar.gz
+        pip install --upgrade pip
+        pip install -r nmaci-$BRANCH/requirements.txt
+        mv nmaci-$BRANCH/scripts/ ci/
+        rm -r nmaci-$BRANCH $BRANCH.tar.gz
+
+    - name: Ignore ci directory
+      shell: bash
+      run: echo "ci/" >> .gitignore

.github/actions/setup-python-env/action.yml

@@ -0,0 +1,32 @@
+name: 'Setup Python Environment'
+description: 'Set up Python with caching and install base dependencies'
+
+inputs:
+  python-version:
+    description: 'Python version to install'
+    required: false
+    default: '3.10'
+
+outputs:
+  cache-hit:
+    description: 'Whether pip cache was hit'
+    value: ${{ steps.setup-python.outputs.cache-hit }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python
+      id: setup-python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+        cache-dependency-path: 'requirements.txt'
+
+    - name: Upgrade pip
+      shell: bash
+      run: pip install --upgrade pip wheel
+
+    - name: Install requirements
+      shell: bash
+      run: pip install -r requirements.txt

.github/actions/setup-rendering-deps/action.yml

@@ -0,0 +1,55 @@
+name: 'Setup Rendering Dependencies'
+description: 'Install fonts, backend libraries, and graphviz for notebook rendering'
+
+inputs:
+  skip-fonts:
+    description: 'Skip XKCD font installation'
+    required: false
+    default: 'false'
+  skip-backend:
+    description: 'Skip backend libraries'
+    required: false
+    default: 'false'
+  skip-graphviz:
+    description: 'Skip graphviz installation'
+    required: false
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Cache APT packages
+      id: cache-apt
+      uses: actions/cache@v3
+      with:
+        path: |
+          /var/cache/apt/archives
+          /usr/share/fonts/truetype/humor-sans
+        key: apt-packages-${{ runner.os }}-fonts-backend-graphviz-v2
+        restore-keys: apt-packages-${{ runner.os }}-
+
+    - name: Install XKCD fonts
+      if: ${{ inputs.skip-fonts != 'true' }}
+      shell: bash
+      run: |
+        if [ ! -f /usr/share/fonts/truetype/humor-sans/Humor-Sans.ttf ]; then
+          sudo apt-get update -yq
+          wget -q http://archive.ubuntu.com/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-4_all.deb
+          sudo dpkg -i --force-all fonts-humor-sans_1.0-4_all.deb <<< 'yes' 2>/dev/null || true
+          sudo apt install -fy
+          rm -f fonts-humor-sans_1.0-4_all.deb $HOME/.matplotlib/fontList.cache
+        else
+          echo "XKCD fonts already cached"
+        fi
+
+    - name: Install backend libraries
+      if: ${{ inputs.skip-backend != 'true' }}
+      shell: bash
+      run: |
+        sudo apt-get update -yq || true
+        sudo apt-get install -y libglew-dev libglfw3 ffmpeg
+        echo "MUJOCO_GL=egl" >> $GITHUB_ENV
+
+    - name: Install Graphviz
+      if: ${{ inputs.skip-graphviz != 'true' }}
+      uses: tlylt/install-graphviz@v1

.github/workflows/notebook-pr.yaml

@@ -27,53 +27,25 @@ jobs:
           readonly local msg=$(git log -1 --pretty=format:"%s")
           echo "COMMIT_MESSAGE=$msg" >> $GITHUB_ENV
 
-      - name: Set up Python
+      - name: Setup Python environment
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
+        uses: ./.github/actions/setup-python-env
 
-      - name: Install CI tools
-        if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        run: |
-          BRANCH=`python -c 'import os, re; m = re.search(r"nmaci:([\w-]+)", os.environ["COMMIT_MESSAGE"]); print("main" if m is None else m.group(1))'`
-          # NOTE: might enventually change this back if we integrate changes
-          wget https://github.com/neuromatch/nmaci/archive/refs/heads/$BRANCH.tar.gz
-          tar -xzf $BRANCH.tar.gz
-          pip install --upgrade pip
-          pip install -r nmaci-$BRANCH/requirements.txt
-          mv nmaci-$BRANCH/scripts/ ci/
-          rm -r nmaci-$BRANCH
-          echo ci/ >> .gitignore
-
-      - name: Install dependencies
+      - name: Install additional dependencies
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
         run: |
-          python -m pip install --upgrade pip wheel
-          pip install -r requirements.txt
-          pip install jupyter_client==7.3.5  # downgrade jupyter-client to fix hangs
-          pip install jinja2==3.1.2 # to fix pandas dep
+          pip install jupyter_client==7.3.5
+          pip install jinja2==3.1.2
 
-      - name: Install XKCD fonts
+      - name: Setup CI tools
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        run: |
-          sudo apt-get update -yq
-          wget http://archive.ubuntu.com/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-4_all.deb
-          sudo dpkg -i --force-all fonts-humor-sans_1.0-4_all.deb <<< 'yes'
-          sudo apt install -fy
-          rm -f $HOME/.matplotlib/fontList.cache
+        uses: ./.github/actions/setup-ci-tools
+        with:
+          commit-message: ${{ env.COMMIT_MESSAGE }}
 
-      - name: Install backend libs
+      - name: Setup rendering dependencies
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        run: |
-          # required for macrocircuits project
-          sudo apt-get install -y libglew-dev
-          sudo apt-get install -y libglfw3
-          sudo apt install -y ffmpeg
-          echo "MUJOCO_GL=egl" >> $GITHUB_ENV
-
-      - name: Install Graphviz
-        uses: tlylt/install-graphviz@v1
+        uses: ./.github/actions/setup-rendering-deps
 
       - name: Get changed files
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
@@ -86,8 +58,20 @@ jobs:
             echo "$file was changed."
           done
 
-      - name: Process notebooks
+      - name: Check for notebook changes
+        id: notebook-check
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
+        run: |
+          CHANGED="${{ steps.changed-files.outputs.all_changed_files }}"
+          if [[ ! "$CHANGED" =~ \.ipynb ]]; then
+            echo "has_notebooks=false" >> $GITHUB_OUTPUT
+            echo "No notebooks changed, skipping processing"
+          else
+            echo "has_notebooks=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Process notebooks
+        if: "!contains(env.COMMIT_MESSAGE, 'skip ci') && steps.notebook-check.outputs.has_notebooks == 'true'"
         id: process_notebooks
         run: |
           branch=${{ github.event.pull_request.head.ref }}
@@ -109,8 +93,18 @@ jobs:
       #   with:
       #     path: comment.txt
 
-      - name: Update READMEs
+      - name: Check if materials.yml changed
+        id: materials-check
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
+        run: |
+          if [[ "${{ steps.changed-files.outputs.all_changed_files }}" =~ materials\.yml ]]; then
+            echo "materials_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "materials_changed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Update READMEs
+        if: "!contains(env.COMMIT_MESSAGE, 'skip ci') && steps.materials-check.outputs.materials_changed == 'true'"
         run: python ci/generate_tutorial_readmes.py
 
       - name: Remove unreferenced derivatives

.github/workflows/publish-book.yml

@@ -29,50 +29,20 @@ jobs:
           readonly local msg=$(git log -1 --pretty=format:"%s")
           echo "COMMIT_MESSAGE=$msg" >> $GITHUB_ENV
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-python-env
 
-      - name: Install CI tools
-        run: |
-          BRANCH=`python -c 'import os, re; m = re.search(r"nmaci:([\w-]+)", os.environ["COMMIT_MESSAGE"]); print("main" if m is None else m.group(1))'`
-          # NOTE: might enventually change this back if we integrate changes
-          wget https://github.com/$ORG/nmaci/archive/refs/heads/$BRANCH.tar.gz
-          tar -xzf $BRANCH.tar.gz
-          pip install --upgrade pip
-          pip install -r nmaci-$BRANCH/requirements.txt
-          mv nmaci-$BRANCH/scripts/ ci/
-          rm -r nmaci-$BRANCH
-          rm -r $BRANCH.tar.gz
-          echo ci/ >> .gitignore
+      - name: Install Jupyter Book
+        run: pip install jupyter-book==0.14.0 ghp-import
 
-      - name: Install dependencies
-        run: |
-          pip install -r requirements.txt
-          pip install jupyter-book==0.14.0 ghp-import
-          # pip install jupyter_client==7.3.5  # downgrade jupyter-client to fix hangs
+      - name: Setup CI tools
+        uses: ./.github/actions/setup-ci-tools
+        with:
+          commit-message: ${{ env.COMMIT_MESSAGE }}
 
-      - name: Install XKCD fonts
-        if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        run: |
-          sudo apt-get update -yq
-          wget http://archive.ubuntu.com/ubuntu/pool/universe/f/fonts-humor-sans/fonts-humor-sans_1.0-4_all.deb
-          sudo dpkg -i --force-all fonts-humor-sans_1.0-4_all.deb <<< 'yes'
-          sudo apt install -fy
-          rm -f $HOME/.matplotlib/fontList.cache
-
-      - name: Install backend libs
+      - name: Setup rendering dependencies
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"
-        run: |
-          # required for macrocircuits project
-          sudo apt-get install -y libglew-dev
-          sudo apt-get install -y libglfw3
-          sudo apt install -y ffmpeg
-          echo "MUJOCO_GL=egl" >> $GITHUB_ENV
-
-      - name: Install Graphviz
-        uses: tlylt/install-graphviz@v1
+        uses: ./.github/actions/setup-rendering-deps
 
       # - name: Copy tutorials from precourse repo
       #   if: "!contains(env.COMMIT_MESSAGE, 'skip precourse')"
@@ -87,6 +57,19 @@ jobs:
       #     rm -r precourse-$BRANCH
       #     rm -r $BRANCH.tar.gz
 
+      - name: Get date for cache rotation
+        id: cache-date
+        run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
+
+      - name: Cache Jupyter execution
+        uses: actions/cache@v3
+        with:
+          path: book/.jupyter-cache
+          key: jupyter-exec-${{ steps.cache-date.outputs.date }}-${{ hashFiles('tutorials/**/*.ipynb', 'requirements.txt') }}
+          restore-keys: |
+            jupyter-exec-${{ steps.cache-date.outputs.date }}-
+            jupyter-exec-
+
       - name: Build student book
         run: |
           python ci/generate_book.py student